doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
// Doppler-side TSIR boundary-point fixture writer.
|
|
2
|
+
//
|
|
3
|
+
// Builds rung-5 frozen-Doppler-reference fixtures for the Doe rung
|
|
4
|
+
// ladder (see /home/x/deco/doe/docs/cerebras-north-star.md and
|
|
5
|
+
// /home/x/deco/doe/config/doe-frozen-doppler-reference.schema.json).
|
|
6
|
+
// Captures activation snapshots at the four TSIR boundary points
|
|
7
|
+
// (`post_rmsnorm`, `post_qkv`, `post_attn`, `post_ffn`) during a
|
|
8
|
+
// reference inference, writes them as `.npy` files keyed by layer
|
|
9
|
+
// index and probe-point name, and emits a manifest the Doe
|
|
10
|
+
// validator (bench/tools/validate_frozen_doppler_reference.py) can
|
|
11
|
+
// consume.
|
|
12
|
+
//
|
|
13
|
+
// Triggered when `operatorDiagnostics.tsirFixture` is set in the
|
|
14
|
+
// inference context. The CLI flag --tsir-fixture-dir on
|
|
15
|
+
// tools/run-program-bundle-reference.js plumbs it in.
|
|
16
|
+
//
|
|
17
|
+
// Stage name mapping (Doppler stage -> Doe TSIR boundary):
|
|
18
|
+
// post_input_norm -> post_rmsnorm
|
|
19
|
+
// linear_qkv_proj -> post_qkv (fused QKV path; single matmul)
|
|
20
|
+
// q_proj+k_proj+v_proj -> post_qkv (split QKV path; concatenated along
|
|
21
|
+
// feature axis as Q∥K∥V at drain time
|
|
22
|
+
// for Gemma 4 31B and similar models)
|
|
23
|
+
// post_attn -> post_attn
|
|
24
|
+
// layer_out -> post_ffn
|
|
25
|
+
// Additional debug-only stage mappings (post_qproj, post_kproj, post_vproj,
|
|
26
|
+
// post_qnorm, post_knorm, post_vnorm, attn_core_out, post_oproj, pre_ffn,
|
|
27
|
+
// ffn_mlp_out) emit per-stage .npy files at non-schema names; the Doe-side
|
|
28
|
+
// builder ignores them, but they are useful for parity bisection.
|
|
29
|
+
|
|
30
|
+
// Use dynamic import for `node:fs/promises` and `node:path` so this
|
|
31
|
+
// module loads cleanly in the browser bundle too (where Node builtins
|
|
32
|
+
// are unavailable). The fixture writer is a no-op in browser context;
|
|
33
|
+
// it only emits files when invoked from the Node-side harness.
|
|
34
|
+
import { readBufferSlice, acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
35
|
+
import { getDevice } from '../../../gpu/device.js';
|
|
36
|
+
|
|
37
|
+
let _nodeFsPromise = null;
|
|
38
|
+
async function getNodeFs() {
|
|
39
|
+
if (_nodeFsPromise) return _nodeFsPromise;
|
|
40
|
+
_nodeFsPromise = (async () => {
|
|
41
|
+
try {
|
|
42
|
+
const fs = await import('node:fs/promises');
|
|
43
|
+
const path = await import('node:path');
|
|
44
|
+
return { fs: fs.default ?? fs, path: path.default ?? path };
|
|
45
|
+
} catch {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
48
|
+
})();
|
|
49
|
+
return _nodeFsPromise;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const STAGE_TO_TSIR = {
|
|
53
|
+
post_input_norm: 'post_rmsnorm',
|
|
54
|
+
// Two QKV-projection paths exist in Doppler:
|
|
55
|
+
// - Fused: `linear_qkv_proj` (linear-attention.js variants)
|
|
56
|
+
// - Split per-projection: `q_proj` (used by standard attention,
|
|
57
|
+
// including Gemma 4)
|
|
58
|
+
// Map both to post_qkv. When both fire (which doesn't happen for
|
|
59
|
+
// any current model), the q_proj snapshot wins because it's
|
|
60
|
+
// overwritten last; that's fine — both name the same TSIR boundary
|
|
61
|
+
// and hash equivalence checks tolerate the asymmetry of using just
|
|
62
|
+
// the Q projection (the boundary is "post Q+K+V projection"; one
|
|
63
|
+
// tensor captures that point in the per-block emit).
|
|
64
|
+
linear_qkv_proj: 'post_qkv',
|
|
65
|
+
q_proj: 'post_qproj',
|
|
66
|
+
k_proj: 'post_kproj',
|
|
67
|
+
v_proj: 'post_vproj',
|
|
68
|
+
q_norm: 'post_qnorm',
|
|
69
|
+
k_norm: 'post_knorm',
|
|
70
|
+
v_norm: 'post_vnorm',
|
|
71
|
+
attn_core_out: 'attn_core_out',
|
|
72
|
+
attn_out: 'post_oproj',
|
|
73
|
+
post_attn: 'post_attn',
|
|
74
|
+
ffn_in: 'pre_ffn',
|
|
75
|
+
ffn_out: 'ffn_mlp_out',
|
|
76
|
+
layer_out: 'post_ffn',
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
export const TSIR_BOUNDARY_STAGES = Object.keys(STAGE_TO_TSIR);
|
|
80
|
+
|
|
81
|
+
export function mapStageToTsirBoundary(stage) {
|
|
82
|
+
return STAGE_TO_TSIR[stage] ?? null;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Track a per-layer q/k/v projection partial in the fixture-level Map and
|
|
86
|
+
// synthesize `post_qkv.npy` once all three are present. Called by both
|
|
87
|
+
// the synchronous write path of `maybeWriteFixtureSnapshot` AND the
|
|
88
|
+
// recorder-deferred drain path so split-q/k/v models always end up with
|
|
89
|
+
// the schema-binding `post_qkv` probe regardless of how the per-stage
|
|
90
|
+
// captures arrived. Returns a record describing the action (write,
|
|
91
|
+
// pending more partials, or write failure) so callers can append it to
|
|
92
|
+
// the fixture's records list.
|
|
93
|
+
async function trackQkvPartialAndMaybeWrite(tsirFixture, stage, layerIdx, numTokens, hiddenSize, arr) {
|
|
94
|
+
const qkvByLayer = (tsirFixture.qkvByLayer ??= new Map());
|
|
95
|
+
let entry = qkvByLayer.get(layerIdx);
|
|
96
|
+
if (!entry) {
|
|
97
|
+
entry = { numTokens, q: null, k: null, v: null, qHidden: 0, kHidden: 0, vHidden: 0 };
|
|
98
|
+
qkvByLayer.set(layerIdx, entry);
|
|
99
|
+
}
|
|
100
|
+
if (stage === 'q_proj') { entry.q = arr; entry.qHidden = hiddenSize; }
|
|
101
|
+
else if (stage === 'k_proj') { entry.k = arr; entry.kHidden = hiddenSize; }
|
|
102
|
+
else if (stage === 'v_proj') { entry.v = arr; entry.vHidden = hiddenSize; }
|
|
103
|
+
if (!entry.q || !entry.k || !entry.v) {
|
|
104
|
+
return { stage: 'q_proj+k_proj+v_proj', tsirStage: 'post_qkv', layerIdx, written: false, note: 'pending-other-projections' };
|
|
105
|
+
}
|
|
106
|
+
const node = await getNodeFs();
|
|
107
|
+
if (!node) {
|
|
108
|
+
return { stage: 'q_proj+k_proj+v_proj', tsirStage: 'post_qkv', layerIdx, written: false, note: 'node-fs-unavailable-in-browser' };
|
|
109
|
+
}
|
|
110
|
+
const { path } = node;
|
|
111
|
+
const totalHidden = entry.qHidden + entry.kHidden + entry.vHidden;
|
|
112
|
+
const merged = new Float32Array(entry.numTokens * totalHidden);
|
|
113
|
+
for (let t = 0; t < entry.numTokens; t++) {
|
|
114
|
+
const dstBase = t * totalHidden;
|
|
115
|
+
merged.set(entry.q.subarray(t * entry.qHidden, (t + 1) * entry.qHidden), dstBase);
|
|
116
|
+
merged.set(entry.k.subarray(t * entry.kHidden, (t + 1) * entry.kHidden), dstBase + entry.qHidden);
|
|
117
|
+
merged.set(entry.v.subarray(t * entry.vHidden, (t + 1) * entry.vHidden), dstBase + entry.qHidden + entry.kHidden);
|
|
118
|
+
}
|
|
119
|
+
const layerDir = path.join(tsirFixture.dir, `layer_${layerIdx}`);
|
|
120
|
+
const filePath = path.join(layerDir, 'post_qkv.npy');
|
|
121
|
+
qkvByLayer.delete(layerIdx);
|
|
122
|
+
try {
|
|
123
|
+
const info = await writeNpyF32(filePath, [entry.numTokens, totalHidden], merged);
|
|
124
|
+
return {
|
|
125
|
+
stage: 'q_proj+k_proj+v_proj',
|
|
126
|
+
tsirStage: 'post_qkv',
|
|
127
|
+
layerIdx,
|
|
128
|
+
filePath,
|
|
129
|
+
shape: [entry.numTokens, totalHidden],
|
|
130
|
+
dtype: 'float32',
|
|
131
|
+
...info,
|
|
132
|
+
written: true,
|
|
133
|
+
note: `synthesized from q/k/v concat (qHidden=${entry.qHidden}, kHidden=${entry.kHidden}, vHidden=${entry.vHidden})`,
|
|
134
|
+
};
|
|
135
|
+
} catch (e) {
|
|
136
|
+
return { stage: 'q_proj+k_proj+v_proj', tsirStage: 'post_qkv', layerIdx, written: false, note: `qkv-concat-write-failed: ${e?.message ?? e}` };
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// numpy .npy v1.0 writer for f32. Shape is a flat array of ints.
|
|
141
|
+
// data is a Float32Array (already-decoded; caller handles f16->f32).
|
|
142
|
+
export async function writeNpyF32(filePath, shape, data) {
|
|
143
|
+
const node = await getNodeFs();
|
|
144
|
+
if (!node) throw new Error('writeNpyF32 requires Node fs/path (not available in browser)');
|
|
145
|
+
const { fs, path } = node;
|
|
146
|
+
const magic = Uint8Array.of(0x93, 0x4e, 0x55, 0x4d, 0x50, 0x59); // \x93NUMPY
|
|
147
|
+
const ver = Uint8Array.of(0x01, 0x00); // major=1 minor=0
|
|
148
|
+
const shapeStr = shape.length === 1 ? `(${shape[0]},)` : `(${shape.join(', ')})`;
|
|
149
|
+
let header = `{'descr': '<f4', 'fortran_order': False, 'shape': ${shapeStr}, }`;
|
|
150
|
+
// Pad header so the data starts at a 64-byte boundary.
|
|
151
|
+
// Total header = magic(6) + ver(2) + headerLen(2) + headerText
|
|
152
|
+
const baseLen = 6 + 2 + 2 + header.length + 1; // +1 for trailing \n
|
|
153
|
+
const padding = (64 - (baseLen % 64)) % 64;
|
|
154
|
+
header = header + ' '.repeat(padding) + '\n';
|
|
155
|
+
const headerBytes = new TextEncoder().encode(header);
|
|
156
|
+
if (headerBytes.length > 0xFFFF) throw new Error('npy header too large for v1.0');
|
|
157
|
+
const headerLen = Uint8Array.of(headerBytes.length & 0xFF, (headerBytes.length >> 8) & 0xFF);
|
|
158
|
+
|
|
159
|
+
const payload = new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
|
|
160
|
+
const out = new Uint8Array(magic.length + ver.length + headerLen.length + headerBytes.length + payload.length);
|
|
161
|
+
let off = 0;
|
|
162
|
+
out.set(magic, off); off += magic.length;
|
|
163
|
+
out.set(ver, off); off += ver.length;
|
|
164
|
+
out.set(headerLen, off); off += headerLen.length;
|
|
165
|
+
out.set(headerBytes, off); off += headerBytes.length;
|
|
166
|
+
out.set(payload, off);
|
|
167
|
+
|
|
168
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
169
|
+
await fs.writeFile(filePath, out);
|
|
170
|
+
return { byteLength: out.length, payloadByteLength: payload.length };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// f16 -> f32 conversion for buffer readback.
|
|
174
|
+
function f16ToF32(h) {
|
|
175
|
+
const sign = (h >> 15) & 0x1;
|
|
176
|
+
const exp = (h >> 10) & 0x1f;
|
|
177
|
+
const mant = h & 0x3ff;
|
|
178
|
+
if (exp === 0) return (sign ? -1 : 1) * Math.pow(2, -14) * (mant / 1024);
|
|
179
|
+
if (exp === 31) return mant === 0 ? (sign ? -Infinity : Infinity) : NaN;
|
|
180
|
+
return (sign ? -1 : 1) * Math.pow(2, exp - 15) * (1 + mant / 1024);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function decodeBuffer(arrayBuffer, dtype) {
|
|
184
|
+
if (dtype === 'f16') {
|
|
185
|
+
const u16 = new Uint16Array(arrayBuffer);
|
|
186
|
+
const out = new Float32Array(u16.length);
|
|
187
|
+
for (let i = 0; i < u16.length; i++) out[i] = f16ToF32(u16[i]);
|
|
188
|
+
return out;
|
|
189
|
+
}
|
|
190
|
+
return new Float32Array(arrayBuffer);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Read a Doppler buffer (CPU Float32Array OR WebGPU GPUBuffer) into
|
|
194
|
+
// a Float32Array of the expected element count. Caller passes the
|
|
195
|
+
// dtype so f16 can be widened. For GPU buffers we use the buffer's
|
|
196
|
+
// MAP_READ availability; if mapping fails we return null and the
|
|
197
|
+
// fixture write is silently skipped (caller logs).
|
|
198
|
+
async function readBufferToFloat32(buffer, dtype, expectedElems) {
|
|
199
|
+
if (buffer instanceof Float32Array) {
|
|
200
|
+
if (buffer.length < expectedElems) return null;
|
|
201
|
+
return buffer.slice(0, expectedElems);
|
|
202
|
+
}
|
|
203
|
+
// WebGPU GPUBuffer path: use the buffer-pool's existing readback
|
|
204
|
+
// helper so staging buffers + alignment + readback gates all match
|
|
205
|
+
// the rest of Doppler's instrumentation.
|
|
206
|
+
if (buffer && typeof buffer.size === 'number') {
|
|
207
|
+
try {
|
|
208
|
+
const bytesPerElem = dtype === 'f16' ? 2 : 4;
|
|
209
|
+
const bytes = expectedElems * bytesPerElem;
|
|
210
|
+
const arrayBuffer = await readBufferSlice(buffer, 0, bytes);
|
|
211
|
+
if (!arrayBuffer || arrayBuffer.byteLength === 0) return null;
|
|
212
|
+
return decodeBuffer(arrayBuffer, dtype);
|
|
213
|
+
} catch (e) {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return null;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Record a probe snapshot if the (stage, layerIdx) pair matches the
|
|
221
|
+
// fixture writer's filter. Returns a record describing what was
|
|
222
|
+
// written, or null if skipped. Errors are caught and returned as a
|
|
223
|
+
// note on the record so the caller can log without crashing the run.
|
|
224
|
+
export async function maybeWriteFixtureSnapshot(stage, buffer, options) {
|
|
225
|
+
const { tsirFixture, layerIdx, numTokens, hiddenSize, dtype = 'f32', recorder } = options;
|
|
226
|
+
if (!tsirFixture || !tsirFixture.dir) return null;
|
|
227
|
+
const tsirStage = mapStageToTsirBoundary(stage);
|
|
228
|
+
if (!tsirStage) return null;
|
|
229
|
+
const layerFilter = tsirFixture.layerFilter ?? null;
|
|
230
|
+
if (Array.isArray(layerFilter) && !layerFilter.includes(layerIdx)) return null;
|
|
231
|
+
// Skip decode-step writes (numTokens === 1) so prefill snapshots aren't
|
|
232
|
+
// overwritten by single-token decode passes. Fixture semantics expect a
|
|
233
|
+
// multi-token prefill capture.
|
|
234
|
+
if (tsirFixture.prefillOnly !== false && numTokens === 1) return null;
|
|
235
|
+
// When a prefill recorder is in-flight, the buffer doesn't have the
|
|
236
|
+
// recorded ops applied yet — the staging copy would return zeros from
|
|
237
|
+
// the pool. Defer the snapshot until the recorder has been submitted
|
|
238
|
+
// by queuing the metadata; drainPendingTsirReads() processes the queue
|
|
239
|
+
// after recorder.submitAndWait().
|
|
240
|
+
if (recorder) {
|
|
241
|
+
// Copy the in-flight buffer's bytes to a freshly-acquired carry buffer
|
|
242
|
+
// using the same recorder's encoder. When the recorder submits, the
|
|
243
|
+
// carry buffer holds the values at this probe point. Then drain after
|
|
244
|
+
// submitAndWait().
|
|
245
|
+
const expectedBytes = numTokens * hiddenSize * (dtype === 'f16' ? 2 : 4);
|
|
246
|
+
const alignedBytes = Math.ceil(expectedBytes / 4) * 4;
|
|
247
|
+
if (!buffer || typeof buffer.size !== 'number') return null;
|
|
248
|
+
if (typeof recorder.getEncoder !== 'function') return null;
|
|
249
|
+
const carry = acquireBuffer(
|
|
250
|
+
Math.max(alignedBytes, 256),
|
|
251
|
+
typeof buffer.usage === 'number' ? buffer.usage : undefined,
|
|
252
|
+
`tsir_carry_${tsirStage}_L${layerIdx}`
|
|
253
|
+
);
|
|
254
|
+
try {
|
|
255
|
+
recorder.getEncoder().copyBufferToBuffer(buffer, 0, carry, 0, alignedBytes);
|
|
256
|
+
} catch (e) {
|
|
257
|
+
releaseBuffer(carry);
|
|
258
|
+
return { stage, tsirStage, layerIdx, written: false, note: `carry-copy-failed: ${e?.message ?? e}` };
|
|
259
|
+
}
|
|
260
|
+
const pending = (tsirFixture.pendingReads ??= []);
|
|
261
|
+
pending.push({ stage, tsirStage, layerIdx, numTokens, hiddenSize, dtype, carry, alignedBytes });
|
|
262
|
+
return { stage, tsirStage, layerIdx, written: false, note: 'pending-recorder-submit' };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const expectedElems = numTokens * hiddenSize;
|
|
266
|
+
const data = await readBufferToFloat32(buffer, dtype, expectedElems);
|
|
267
|
+
if (!data) {
|
|
268
|
+
return {
|
|
269
|
+
stage,
|
|
270
|
+
tsirStage,
|
|
271
|
+
layerIdx,
|
|
272
|
+
written: false,
|
|
273
|
+
note: 'buffer-readback-failed',
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
const node = await getNodeFs();
|
|
278
|
+
if (!node) {
|
|
279
|
+
return {
|
|
280
|
+
stage,
|
|
281
|
+
tsirStage,
|
|
282
|
+
layerIdx,
|
|
283
|
+
written: false,
|
|
284
|
+
note: 'node-fs-unavailable-in-browser',
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
const { path } = node;
|
|
288
|
+
const layerDir = path.join(tsirFixture.dir, `layer_${layerIdx}`);
|
|
289
|
+
const filePath = path.join(layerDir, `${tsirStage}.npy`);
|
|
290
|
+
let perStageRecord;
|
|
291
|
+
try {
|
|
292
|
+
const info = await writeNpyF32(filePath, [numTokens, hiddenSize], data);
|
|
293
|
+
perStageRecord = {
|
|
294
|
+
stage,
|
|
295
|
+
tsirStage,
|
|
296
|
+
layerIdx,
|
|
297
|
+
filePath,
|
|
298
|
+
shape: [numTokens, hiddenSize],
|
|
299
|
+
dtype: 'float32',
|
|
300
|
+
...info,
|
|
301
|
+
written: true,
|
|
302
|
+
};
|
|
303
|
+
} catch (e) {
|
|
304
|
+
return {
|
|
305
|
+
stage,
|
|
306
|
+
tsirStage,
|
|
307
|
+
layerIdx,
|
|
308
|
+
written: false,
|
|
309
|
+
note: `write-failed: ${e?.message ?? e}`,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
// Synthesize post_qkv.npy on the fly when split-q/k/v projections are
|
|
313
|
+
// captured via the synchronous path (recorder undefined). Models with
|
|
314
|
+
// a fused linear_qkv_proj path emit post_qkv directly; this branch
|
|
315
|
+
// handles the split-projection case (Gemma 4 31B etc.).
|
|
316
|
+
if (stage === 'q_proj' || stage === 'k_proj' || stage === 'v_proj') {
|
|
317
|
+
const qkvRecord = await trackQkvPartialAndMaybeWrite(
|
|
318
|
+
tsirFixture, stage, layerIdx, numTokens, hiddenSize, data,
|
|
319
|
+
);
|
|
320
|
+
if (qkvRecord && qkvRecord.written && Array.isArray(tsirFixture.records)) {
|
|
321
|
+
tsirFixture.records.push(qkvRecord);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
return perStageRecord;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
// Drain pending TSIR snapshots that were queued during a prefill recorder
|
|
329
|
+
// pass. Call this after `recorder.submitAndWait()` so the carry buffers
|
|
330
|
+
// hold valid recorded values. Releases each carry buffer after readback.
|
|
331
|
+
//
|
|
332
|
+
// After per-stage writes complete, this also synthesizes the schema-binding
|
|
333
|
+
// `post_qkv.npy` for layers that have all three of `post_qproj`,
|
|
334
|
+
// `post_kproj`, `post_vproj` captured. The Doe schema
|
|
335
|
+
// (config/doe-frozen-doppler-reference.schema.json) requires the probe-point
|
|
336
|
+
// name `post_qkv`. For models with split q_proj/k_proj/v_proj (Gemma 4 31B,
|
|
337
|
+
// etc.), we capture each individually for debug introspection AND emit the
|
|
338
|
+
// concatenated Q∥K∥V tensor along the feature axis as `post_qkv.npy` so the
|
|
339
|
+
// Doe-side rung-5 builder + validator + rung-7 oracle can consume it. The
|
|
340
|
+
// fused-QKV path (`linear_qkv_proj`) writes to `post_qkv.npy` directly and
|
|
341
|
+
// is not re-synthesized.
|
|
342
|
+
export async function drainPendingTsirReads(tsirFixture) {
|
|
343
|
+
if (!tsirFixture || !tsirFixture.dir) return [];
|
|
344
|
+
const pending = tsirFixture.pendingReads;
|
|
345
|
+
if (!pending || pending.length === 0) return [];
|
|
346
|
+
const node = await getNodeFs();
|
|
347
|
+
const records = [];
|
|
348
|
+
for (const item of pending) {
|
|
349
|
+
const { stage, tsirStage, layerIdx, numTokens, hiddenSize, dtype, carry, alignedBytes } = item;
|
|
350
|
+
let arr = null;
|
|
351
|
+
let staging = null;
|
|
352
|
+
let mapped = false;
|
|
353
|
+
try {
|
|
354
|
+
const device = getDevice();
|
|
355
|
+
if (!device) {
|
|
356
|
+
records.push({ stage, tsirStage, layerIdx, written: false, note: 'no-device' });
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
staging = device.createBuffer({
|
|
360
|
+
size: alignedBytes,
|
|
361
|
+
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
362
|
+
label: `tsir_drain_staging_${tsirStage}_L${layerIdx}`,
|
|
363
|
+
});
|
|
364
|
+
const enc = device.createCommandEncoder({ label: `tsir_drain_enc_L${layerIdx}` });
|
|
365
|
+
enc.copyBufferToBuffer(carry, 0, staging, 0, alignedBytes);
|
|
366
|
+
device.queue.submit([enc.finish()]);
|
|
367
|
+
await staging.mapAsync(GPUMapMode.READ);
|
|
368
|
+
mapped = true;
|
|
369
|
+
const ab = staging.getMappedRange(0, alignedBytes).slice(0, numTokens * hiddenSize * (dtype === 'f16' ? 2 : 4));
|
|
370
|
+
arr = decodeBuffer(ab, dtype);
|
|
371
|
+
} catch (e) {
|
|
372
|
+
records.push({ stage, tsirStage, layerIdx, written: false, note: `drain-read-failed: ${e?.message ?? e}` });
|
|
373
|
+
} finally {
|
|
374
|
+
if (mapped && staging) staging.unmap();
|
|
375
|
+
if (staging) staging.destroy();
|
|
376
|
+
releaseBuffer(carry);
|
|
377
|
+
}
|
|
378
|
+
if (!arr) continue;
|
|
379
|
+
if (!node) {
|
|
380
|
+
records.push({ stage, tsirStage, layerIdx, written: false, note: 'node-fs-unavailable-in-browser' });
|
|
381
|
+
continue;
|
|
382
|
+
}
|
|
383
|
+
const { path } = node;
|
|
384
|
+
const layerDir = path.join(tsirFixture.dir, `layer_${layerIdx}`);
|
|
385
|
+
const filePath = path.join(layerDir, `${tsirStage}.npy`);
|
|
386
|
+
try {
|
|
387
|
+
const info = await writeNpyF32(filePath, [numTokens, hiddenSize], arr);
|
|
388
|
+
records.push({ stage, tsirStage, layerIdx, filePath, shape: [numTokens, hiddenSize], dtype: 'float32', ...info, written: true });
|
|
389
|
+
} catch (e) {
|
|
390
|
+
records.push({ stage, tsirStage, layerIdx, written: false, note: `write-failed: ${e?.message ?? e}` });
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
// Synthesize post_qkv.npy on the fly for split-q/k/v projection
|
|
394
|
+
// models — same helper the synchronous write path calls so both
|
|
395
|
+
// recorder-deferred and immediate captures end up with the
|
|
396
|
+
// schema-binding probe.
|
|
397
|
+
if (stage === 'q_proj' || stage === 'k_proj' || stage === 'v_proj') {
|
|
398
|
+
const qkvRecord = await trackQkvPartialAndMaybeWrite(
|
|
399
|
+
tsirFixture, stage, layerIdx, numTokens, hiddenSize, arr,
|
|
400
|
+
);
|
|
401
|
+
if (qkvRecord && qkvRecord.written) records.push(qkvRecord);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
tsirFixture.pendingReads = [];
|
|
405
|
+
if (Array.isArray(tsirFixture.records)) {
|
|
406
|
+
tsirFixture.records.push(...records);
|
|
407
|
+
}
|
|
408
|
+
return records;
|
|
409
|
+
}
|
|
@@ -162,16 +162,6 @@ export interface LayerContext {
|
|
|
162
162
|
} | null;
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
-
/**
|
|
166
|
-
* Layer processing result.
|
|
167
|
-
*/
|
|
168
|
-
export interface LayerResult {
|
|
169
|
-
/** Output hidden states (GPUBuffer or Float32Array) */
|
|
170
|
-
output: GPUBuffer | Float32Array;
|
|
171
|
-
/** Whether output is on GPU */
|
|
172
|
-
isGPU: boolean;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
165
|
/**
|
|
176
166
|
* Sandwich norm detection result.
|
|
177
167
|
*/
|
|
@@ -186,54 +176,6 @@ export interface SandwichNormInfo {
|
|
|
186
176
|
hasPostAttentionNorm: boolean;
|
|
187
177
|
}
|
|
188
178
|
|
|
189
|
-
/**
|
|
190
|
-
* Pipeline context - all state needed for inference operations.
|
|
191
|
-
*
|
|
192
|
-
* This is passed to all pipeline functions, providing access to:
|
|
193
|
-
* - Model configuration
|
|
194
|
-
* - Weight buffers
|
|
195
|
-
* - KV cache
|
|
196
|
-
* - GPU resources
|
|
197
|
-
* - Runtime state
|
|
198
|
-
*/
|
|
199
|
-
export interface PipelineContext {
|
|
200
|
-
/** Parsed model configuration */
|
|
201
|
-
config: ParsedModelConfig;
|
|
202
|
-
|
|
203
|
-
/** Weight buffers map (name -> GPUBuffer | WeightBuffer | Float32Array) */
|
|
204
|
-
weights: Map<string, GPUBuffer | WeightBuffer | Float32Array | PerLayerInputWeights | null>;
|
|
205
|
-
|
|
206
|
-
/** KV cache instance */
|
|
207
|
-
kvCache: KVCacheInterface;
|
|
208
|
-
|
|
209
|
-
/** Current sequence length (tokens processed so far) */
|
|
210
|
-
currentSeqLen: number;
|
|
211
|
-
|
|
212
|
-
/** Whether GPU is available and should be used */
|
|
213
|
-
useGPU: boolean;
|
|
214
|
-
|
|
215
|
-
/** Debug mode flag */
|
|
216
|
-
debug: boolean;
|
|
217
|
-
|
|
218
|
-
/** RoPE cosine frequencies buffer */
|
|
219
|
-
ropeFreqsCos: GPUBuffer | Float32Array | null;
|
|
220
|
-
|
|
221
|
-
/** RoPE sine frequencies buffer */
|
|
222
|
-
ropeFreqsSin: GPUBuffer | Float32Array | null;
|
|
223
|
-
|
|
224
|
-
/** Tokenizer instance (for debug logging) */
|
|
225
|
-
tokenizer?: TokenizerInterface;
|
|
226
|
-
|
|
227
|
-
/** Statistics tracking */
|
|
228
|
-
stats: PipelineStats;
|
|
229
|
-
|
|
230
|
-
/** Batching statistics */
|
|
231
|
-
batchingStats: BatchingStats;
|
|
232
|
-
|
|
233
|
-
/** Optional LoRA adapter */
|
|
234
|
-
lora?: LoRAAdapter | null;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
179
|
/** GPU buffer result for KV cache layer */
|
|
238
180
|
export interface GPUBuffersResult {
|
|
239
181
|
keysGPU?: GPUBuffer;
|
|
@@ -317,20 +259,6 @@ export interface KVCacheInterface {
|
|
|
317
259
|
getGPUBuffers?(layerIdx: number): GPUBuffersResult | null;
|
|
318
260
|
}
|
|
319
261
|
|
|
320
|
-
/**
|
|
321
|
-
* Minimal tokenizer interface for pipeline operations.
|
|
322
|
-
*/
|
|
323
|
-
export interface TokenizerInterface {
|
|
324
|
-
/** Encode text to token IDs */
|
|
325
|
-
encode(text: string): number[];
|
|
326
|
-
|
|
327
|
-
/** Decode token IDs to text */
|
|
328
|
-
decode(ids: number[], skipSpecial?: boolean, clean?: boolean): string;
|
|
329
|
-
|
|
330
|
-
/** Get special token IDs */
|
|
331
|
-
getSpecialTokens?(): { bos?: number; eos?: number; pad?: number };
|
|
332
|
-
}
|
|
333
|
-
|
|
334
262
|
// ============================================================================
|
|
335
263
|
// Generation Types
|
|
336
264
|
// ============================================================================
|
|
@@ -377,6 +305,12 @@ export interface GenerateOptions {
|
|
|
377
305
|
/** Callback for each generated token */
|
|
378
306
|
onToken?: ((tokenId: number, text: string) => void) | null;
|
|
379
307
|
|
|
308
|
+
/** @internal Callback with finalized per-step logits before sampling returns the token. */
|
|
309
|
+
onLogits?: ((logits: Float32Array, context: {
|
|
310
|
+
tokenId: number;
|
|
311
|
+
inputTokenCount: number | null;
|
|
312
|
+
}) => void) | null;
|
|
313
|
+
|
|
380
314
|
/** Custom decode function for debugging */
|
|
381
315
|
decode?: (tokens: number[]) => string;
|
|
382
316
|
|
|
@@ -534,7 +468,7 @@ export interface LayerWeights {
|
|
|
534
468
|
inputNorm: GPUBuffer | Float32Array;
|
|
535
469
|
qProj: LayerWeightBuffer;
|
|
536
470
|
kProj: LayerWeightBuffer;
|
|
537
|
-
vProj
|
|
471
|
+
vProj?: LayerWeightBuffer;
|
|
538
472
|
oProj: LayerWeightBuffer;
|
|
539
473
|
convInProj?: LayerWeightBuffer;
|
|
540
474
|
convKernel?: LayerWeightBuffer;
|
|
@@ -557,6 +491,7 @@ export interface LayerWeights {
|
|
|
557
491
|
// Sandwich norms (Gemma 3)
|
|
558
492
|
preFeedforwardNorm?: GPUBuffer | Float32Array;
|
|
559
493
|
postFeedforwardNorm?: GPUBuffer | Float32Array;
|
|
494
|
+
layerScalar?: Float32Array | null;
|
|
560
495
|
|
|
561
496
|
// MoE
|
|
562
497
|
routerWeight?: GPUBuffer | import('../../../gpu/weight-buffer.js').WeightBuffer | Float32Array;
|
|
@@ -722,11 +657,6 @@ export interface BatchingStats {
|
|
|
722
657
|
*/
|
|
723
658
|
export type { MaybeGPUBuffer } from './buffer-types.js';
|
|
724
659
|
|
|
725
|
-
/**
|
|
726
|
-
* Function to decode token IDs to text (for debug logging).
|
|
727
|
-
*/
|
|
728
|
-
export type DecodeFunction = (ids: number[]) => string;
|
|
729
|
-
|
|
730
660
|
/**
|
|
731
661
|
* RoPE (Rotary Position Embedding) options.
|
|
732
662
|
*/
|
|
@@ -33,15 +33,6 @@ export interface WeightDebugFlags {
|
|
|
33
33
|
normOffsetDebugDone?: boolean;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
// ============================================================================
|
|
37
|
-
// Type Guards
|
|
38
|
-
// ============================================================================
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* Type guard to check if a weight value is a LayerWeights object.
|
|
42
|
-
*/
|
|
43
|
-
export function isLayerWeights(value: unknown): value is LayerWeights;
|
|
44
|
-
|
|
45
36
|
/**
|
|
46
37
|
* Get layer weights from weights map with type narrowing.
|
|
47
38
|
*/
|
|
@@ -50,10 +41,6 @@ export function getLayerWeights(
|
|
|
50
41
|
key: string
|
|
51
42
|
): LayerWeights | null;
|
|
52
43
|
|
|
53
|
-
// ============================================================================
|
|
54
|
-
// Weight Buffer Creation
|
|
55
|
-
// ============================================================================
|
|
56
|
-
|
|
57
44
|
/**
|
|
58
45
|
* Get or create GPU buffer for a weight tensor.
|
|
59
46
|
*/
|
|
@@ -71,40 +58,3 @@ export function getNormWeightBuffer(
|
|
|
71
58
|
config: WeightBufferConfig,
|
|
72
59
|
debugFlags?: WeightDebugFlags
|
|
73
60
|
): GPUBuffer;
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
* Get GPU weight buffer, ensuring it's on GPU.
|
|
77
|
-
*/
|
|
78
|
-
export function getGPUWeightBuffer(
|
|
79
|
-
weight: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | ArrayBuffer,
|
|
80
|
-
label: string
|
|
81
|
-
): GPUBuffer;
|
|
82
|
-
|
|
83
|
-
// ============================================================================
|
|
84
|
-
// Weight Buffer Factory
|
|
85
|
-
// ============================================================================
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* Create weight buffer helper functions bound to a specific config.
|
|
89
|
-
*/
|
|
90
|
-
export function createWeightBufferHelpers(
|
|
91
|
-
config: WeightBufferConfig,
|
|
92
|
-
debugFlags?: WeightDebugFlags
|
|
93
|
-
): {
|
|
94
|
-
getWeightBuffer: (weight: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | ArrayBuffer, label: string) => GPUBuffer | WeightBuffer;
|
|
95
|
-
getNormWeightBuffer: (weight: GPUBuffer | WeightBuffer | Float32Array | ArrayBuffer | CpuWeightBuffer, label: string) => GPUBuffer;
|
|
96
|
-
getGPUWeightBuffer: (weight: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | ArrayBuffer, label: string) => GPUBuffer;
|
|
97
|
-
};
|
|
98
|
-
|
|
99
|
-
// ============================================================================
|
|
100
|
-
// Batch Buffer Tracking
|
|
101
|
-
// ============================================================================
|
|
102
|
-
|
|
103
|
-
/**
|
|
104
|
-
* Buffer tracking for batched command execution.
|
|
105
|
-
*/
|
|
106
|
-
export class BatchBufferTracker {
|
|
107
|
-
track(buffer: GPUBuffer | Float32Array | ArrayBuffer): void;
|
|
108
|
-
getTracked(): GPUBuffer[];
|
|
109
|
-
clear(): void;
|
|
110
|
-
}
|
|
@@ -10,7 +10,7 @@ import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, tagBufferDtype
|
|
|
10
10
|
// ============================================================================
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
function isLayerWeights(value) {
|
|
14
14
|
return value !== null && typeof value === 'object' && !ArrayBuffer.isView(value) && !('getMappedRange' in (value)) && !isWeightBuffer(value) && !isCpuWeightBuffer(value);
|
|
15
15
|
}
|
|
16
16
|
|
|
@@ -112,7 +112,7 @@ export function getNormWeightBuffer(weight, label, config, debugFlags) {
|
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
|
|
115
|
-
|
|
115
|
+
function getGPUWeightBuffer(weight, label) {
|
|
116
116
|
// Handle WeightBuffer by extracting underlying GPUBuffer
|
|
117
117
|
if (isWeightBuffer(weight)) {
|
|
118
118
|
return weight.buffer;
|
|
@@ -126,52 +126,3 @@ export function getGPUWeightBuffer(weight, label) {
|
|
|
126
126
|
return (getWeightBuffer(weight, label));
|
|
127
127
|
}
|
|
128
128
|
|
|
129
|
-
// ============================================================================
|
|
130
|
-
// Weight Buffer Factory
|
|
131
|
-
// ============================================================================
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
export function createWeightBufferHelpers(config, debugFlags) {
|
|
135
|
-
return {
|
|
136
|
-
|
|
137
|
-
getWeightBuffer: (weight, label) =>
|
|
138
|
-
getWeightBuffer(weight, label),
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
getNormWeightBuffer: (weight, label) =>
|
|
142
|
-
getNormWeightBuffer(weight, label, config, debugFlags),
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
getGPUWeightBuffer: (weight, label) =>
|
|
146
|
-
getGPUWeightBuffer(weight, label),
|
|
147
|
-
};
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
// ============================================================================
|
|
151
|
-
// Batch Buffer Tracking
|
|
152
|
-
// ============================================================================
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
export class BatchBufferTracker {
|
|
156
|
-
constructor() {
|
|
157
|
-
|
|
158
|
-
this._buffersToRelease = [];
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
track(buffer) {
|
|
163
|
-
if (isGpuBufferInstance(buffer)) {
|
|
164
|
-
this._buffersToRelease.push(buffer);
|
|
165
|
-
}
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
getTracked() {
|
|
170
|
-
return this._buffersToRelease;
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
clear() {
|
|
175
|
-
this._buffersToRelease = [];
|
|
176
|
-
}
|
|
177
|
-
}
|
|
@@ -126,7 +126,15 @@ export declare class InferencePipeline extends PipelineState {
|
|
|
126
126
|
getMemoryStats(): {
|
|
127
127
|
used: number;
|
|
128
128
|
pool?: { currentBytesAllocated?: number; peakBytesAllocated?: number; activeBuffers?: number; pooledBuffers?: number };
|
|
129
|
-
kvCache?: {
|
|
129
|
+
kvCache?: {
|
|
130
|
+
allocated?: number;
|
|
131
|
+
used?: number;
|
|
132
|
+
seqLen?: number;
|
|
133
|
+
maxSeqLen?: number;
|
|
134
|
+
layout?: string | null;
|
|
135
|
+
kvDtype?: string | null;
|
|
136
|
+
counters?: Record<string, unknown> | null;
|
|
137
|
+
};
|
|
130
138
|
emulation?: EmulationStats;
|
|
131
139
|
};
|
|
132
140
|
|