doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
// Minimal JSON-grammar sampling constraint.
|
|
2
|
+
//
|
|
3
|
+
// Usage:
|
|
4
|
+
// const mask = createJsonGrammarMask({ tokenizer });
|
|
5
|
+
// for await (const text of pipeline.generate(prompt, { logitMaskFn: mask })) { ... }
|
|
6
|
+
//
|
|
7
|
+
// The mask classifies each candidate token against the current parse state and
|
|
8
|
+
// zeroes logits for tokens that would break a valid JSON object output. It
|
|
9
|
+
// deliberately accepts any token that *looks like* it may extend valid JSON —
|
|
10
|
+
// this is a soft constraint; parsing the final output is still required to
|
|
11
|
+
// validate the full payload.
|
|
12
|
+
//
|
|
13
|
+
// Tokenizer assumption: `tokenizer.decode([id])` returns the literal string
|
|
14
|
+
// piece for that token. That's true for BPE tokenizers used by Qwen, Gemma,
|
|
15
|
+
// Llama. The mask is deliberately token-piece based, not character level,
|
|
16
|
+
// because re-encoding the stream to characters per step is prohibitively slow
|
|
17
|
+
// in-browser.
|
|
18
|
+
//
|
|
19
|
+
// What the mask does enforce:
|
|
20
|
+
// - Until the first "{" is emitted, only tokens whose piece contains "{" or
|
|
21
|
+
// whitespace are allowed.
|
|
22
|
+
// - Once inside the object, matched counts of "{" / "}" / "[" / "]" / "\""
|
|
23
|
+
// determine whether we are inside a string; inside a string, any token is
|
|
24
|
+
// allowed except one that closes more brackets than are open.
|
|
25
|
+
// - After a balanced closing "}" is emitted at depth 0, we forbid any
|
|
26
|
+
// further non-whitespace token (forces a clean stop).
|
|
27
|
+
//
|
|
28
|
+
// What the mask does NOT enforce (intentional, to keep the mask cheap):
|
|
29
|
+
// - Key quoting, trailing-comma avoidance, matching " pairs at token level.
|
|
30
|
+
// Post-decode `parseJSONObjectEnvelope` handles these via recovery
|
|
31
|
+
// transformations.
|
|
32
|
+
// - Schema/key whitelists. Callers can layer a stricter mask on top.
|
|
33
|
+
|
|
34
|
+
function pieceAt(tokenizer, tokenId) {
|
|
35
|
+
if (!tokenizer || typeof tokenizer.decode !== "function") return "";
|
|
36
|
+
try {
|
|
37
|
+
// decode(ids, skipSpecial, skipBos)
|
|
38
|
+
return String(tokenizer.decode([tokenId], true, false) || "");
|
|
39
|
+
} catch {
|
|
40
|
+
return "";
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function countRunes(piece, char) {
|
|
45
|
+
let count = 0;
|
|
46
|
+
for (let i = 0; i < piece.length; i += 1) {
|
|
47
|
+
if (piece[i] === char) count += 1;
|
|
48
|
+
}
|
|
49
|
+
return count;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Piece-level flags used by the mask.
|
|
53
|
+
function classifyPiece(piece) {
|
|
54
|
+
const openBraces = countRunes(piece, "{");
|
|
55
|
+
const closeBraces = countRunes(piece, "}");
|
|
56
|
+
const openBrackets = countRunes(piece, "[");
|
|
57
|
+
const closeBrackets = countRunes(piece, "]");
|
|
58
|
+
const quotes = countRunes(piece, "\"");
|
|
59
|
+
return {
|
|
60
|
+
openBraces,
|
|
61
|
+
closeBraces,
|
|
62
|
+
openBrackets,
|
|
63
|
+
closeBrackets,
|
|
64
|
+
quotes,
|
|
65
|
+
isOnlyWhitespace: /^\s*$/.test(piece),
|
|
66
|
+
hasOpenBrace: openBraces > 0,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Create a logit mask function that enforces a soft JSON-object grammar.
|
|
72
|
+
*
|
|
73
|
+
* @param {{ tokenizer?: { decode(ids: number[], skipSpecial?: boolean, skipBos?: boolean): string } | null, cacheBudget?: number }} [opts]
|
|
74
|
+
* @returns {(logits: Float32Array, context: { generatedIds: number[], tokenizer?: unknown, vocabSize?: number }) => void}
|
|
75
|
+
*/
|
|
76
|
+
export function createJsonGrammarMask(opts = {}) {
|
|
77
|
+
const pieceCache = new Map();
|
|
78
|
+
const cacheBudget = Math.max(1024, Math.floor(Number(opts.cacheBudget) || 32768));
|
|
79
|
+
|
|
80
|
+
function cachedPiece(tokenizer, tokenId) {
|
|
81
|
+
if (pieceCache.has(tokenId)) return pieceCache.get(tokenId);
|
|
82
|
+
if (pieceCache.size >= cacheBudget) pieceCache.clear();
|
|
83
|
+
const piece = pieceAt(tokenizer, tokenId);
|
|
84
|
+
pieceCache.set(tokenId, piece);
|
|
85
|
+
return piece;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
let runningOpenBraces = 0;
|
|
89
|
+
let runningOpenBrackets = 0;
|
|
90
|
+
let runningQuotes = 0;
|
|
91
|
+
let sawFirstBrace = false;
|
|
92
|
+
let balancedClose = false;
|
|
93
|
+
let lastGeneratedLen = 0;
|
|
94
|
+
|
|
95
|
+
function reflectEmittedTokens(tokenizer, generatedIds) {
|
|
96
|
+
// Catch up to any tokens committed since we were last called.
|
|
97
|
+
for (let i = lastGeneratedLen; i < generatedIds.length; i += 1) {
|
|
98
|
+
const piece = cachedPiece(tokenizer, generatedIds[i]);
|
|
99
|
+
const shape = classifyPiece(piece);
|
|
100
|
+
runningOpenBraces += shape.openBraces - shape.closeBraces;
|
|
101
|
+
runningOpenBrackets += shape.openBrackets - shape.closeBrackets;
|
|
102
|
+
runningQuotes += shape.quotes;
|
|
103
|
+
if (!sawFirstBrace && shape.hasOpenBrace) sawFirstBrace = true;
|
|
104
|
+
if (sawFirstBrace && runningOpenBraces <= 0 && runningOpenBrackets <= 0) {
|
|
105
|
+
balancedClose = true;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
lastGeneratedLen = generatedIds.length;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return function logitMask(logits, context) {
|
|
112
|
+
const tokenizer = (opts.tokenizer ?? context?.tokenizer) || null;
|
|
113
|
+
if (!tokenizer) return;
|
|
114
|
+
reflectEmittedTokens(tokenizer, Array.isArray(context?.generatedIds) ? context.generatedIds : []);
|
|
115
|
+
const insideString = runningQuotes % 2 === 1;
|
|
116
|
+
const vocabSize = Math.min(logits.length, Number(context?.vocabSize || logits.length));
|
|
117
|
+
for (let id = 0; id < vocabSize; id += 1) {
|
|
118
|
+
if (logits[id] <= -Infinity) continue;
|
|
119
|
+
const piece = cachedPiece(tokenizer, id);
|
|
120
|
+
if (!piece) continue;
|
|
121
|
+
const shape = classifyPiece(piece);
|
|
122
|
+
let ok = true;
|
|
123
|
+
if (!sawFirstBrace) {
|
|
124
|
+
// Before the first "{", only allow whitespace or a piece that opens a brace.
|
|
125
|
+
ok = shape.isOnlyWhitespace || shape.hasOpenBrace;
|
|
126
|
+
} else if (balancedClose) {
|
|
127
|
+
// After the balanced close, only whitespace is allowed (model should stop).
|
|
128
|
+
ok = shape.isOnlyWhitespace;
|
|
129
|
+
} else if (insideString) {
|
|
130
|
+
// Inside a string literal, accept any piece — even ones with braces or
|
|
131
|
+
// brackets — the parser handles them as content.
|
|
132
|
+
ok = true;
|
|
133
|
+
} else {
|
|
134
|
+
// Outside strings: disallow pieces that close MORE brackets than are open.
|
|
135
|
+
if (shape.closeBraces > runningOpenBraces + shape.openBraces) ok = false;
|
|
136
|
+
if (shape.closeBrackets > runningOpenBrackets + shape.openBrackets) ok = false;
|
|
137
|
+
}
|
|
138
|
+
if (!ok) {
|
|
139
|
+
logits[id] = -Infinity;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
// Minimal TSV-grammar sampling constraint.
|
|
2
|
+
//
|
|
3
|
+
// Usage:
|
|
4
|
+
// const mask = createTsvGrammarMask({ tokenizer, fieldsPerLine: 4 });
|
|
5
|
+
// for await (const text of pipeline.generate(prompt, { logitMaskFn: mask })) { ... }
|
|
6
|
+
//
|
|
7
|
+
// Companion to json-grammar-mask.js. Use this when the model is asked to
|
|
8
|
+
// emit a flat tab-separated record stream (one record per line, fields
|
|
9
|
+
// separated by TAB, terminated by NEWLINE) rather than a JSON envelope.
|
|
10
|
+
// Cuts decode tokens by ~30-40% vs. JSON because the model isn't sampling
|
|
11
|
+
// braces/quotes/keys at every position — and Doppler's structured-
|
|
12
|
+
// generation primitive is logit validation only (no token forcing), so
|
|
13
|
+
// JSON's structural overhead is real.
|
|
14
|
+
//
|
|
15
|
+
// What the mask enforces (soft):
|
|
16
|
+
// - Inside a line with fewer than `fieldsPerLine - 1` tabs already
|
|
17
|
+
// emitted, disallow tokens whose piece contains a NEWLINE — the line
|
|
18
|
+
// must hit all field separators before terminating.
|
|
19
|
+
// - Disallow tokens that would push the per-line tab count past
|
|
20
|
+
// `fieldsPerLine - 1`, so the model can't emit five-field rows when
|
|
21
|
+
// four were specified.
|
|
22
|
+
// - Allow any token (including end-of-stream) after a balanced newline
|
|
23
|
+
// when at least one valid line has been emitted.
|
|
24
|
+
//
|
|
25
|
+
// What the mask does NOT enforce (intentional):
|
|
26
|
+
// - Field-content shape. The category/confidence/page-number values
|
|
27
|
+
// are validated post-decode by the consumer (e.g.,
|
|
28
|
+
// parseDelimitedExtraction + validateRedaction in Columbo). Trying
|
|
29
|
+
// to enforce enum prefixes at the BPE token-piece level is brittle
|
|
30
|
+
// because token boundaries don't align with field boundaries.
|
|
31
|
+
// - Empty lines. A trailing blank line is just dropped by the parser.
|
|
32
|
+
//
|
|
33
|
+
// Tokenizer assumption: same as json-grammar-mask.js — `tokenizer.decode([id])`
|
|
34
|
+
// returns the literal piece for that token. True for Qwen, Gemma, Llama BPE.
|
|
35
|
+
|
|
36
|
+
function pieceAt(tokenizer, tokenId) {
|
|
37
|
+
if (!tokenizer || typeof tokenizer.decode !== "function") return "";
|
|
38
|
+
try {
|
|
39
|
+
return String(tokenizer.decode([tokenId], true, false) || "");
|
|
40
|
+
} catch {
|
|
41
|
+
return "";
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function countRunes(piece, char) {
|
|
46
|
+
let count = 0;
|
|
47
|
+
for (let i = 0; i < piece.length; i += 1) {
|
|
48
|
+
if (piece[i] === char) count += 1;
|
|
49
|
+
}
|
|
50
|
+
return count;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Per-piece flags used by the mask. Tracks tab/newline placement so the
|
|
55
|
+
* state machine can decide when a line is complete and whether a token
|
|
56
|
+
* is allowed to advance the schema.
|
|
57
|
+
*/
|
|
58
|
+
function classifyPiece(piece) {
|
|
59
|
+
const tabs = countRunes(piece, "\t");
|
|
60
|
+
const newlines = countRunes(piece, "\n");
|
|
61
|
+
return {
|
|
62
|
+
tabs,
|
|
63
|
+
newlines,
|
|
64
|
+
isOnlyWhitespace: /^\s*$/.test(piece),
|
|
65
|
+
hasNewline: newlines > 0,
|
|
66
|
+
hasTab: tabs > 0,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Create a logit mask function that enforces a soft TSV grammar.
|
|
72
|
+
*
|
|
73
|
+
* @param {{
|
|
74
|
+
* tokenizer?: { decode(ids: number[], skipSpecial?: boolean, skipBos?: boolean): string } | null,
|
|
75
|
+
* fieldsPerLine?: number,
|
|
76
|
+
* cacheBudget?: number,
|
|
77
|
+
* }} [opts]
|
|
78
|
+
* @returns {(logits: Float32Array, context: { generatedIds: number[], tokenizer?: unknown, vocabSize?: number }) => void}
|
|
79
|
+
*/
|
|
80
|
+
export function createTsvGrammarMask(opts = {}) {
|
|
81
|
+
const fieldsPerLine = Math.max(2, Math.floor(Number(opts.fieldsPerLine) || 4));
|
|
82
|
+
const tabsPerLineMax = fieldsPerLine - 1;
|
|
83
|
+
const pieceCache = new Map();
|
|
84
|
+
const cacheBudget = Math.max(1024, Math.floor(Number(opts.cacheBudget) || 32768));
|
|
85
|
+
|
|
86
|
+
function cachedPiece(tokenizer, tokenId) {
|
|
87
|
+
if (pieceCache.has(tokenId)) return pieceCache.get(tokenId);
|
|
88
|
+
if (pieceCache.size >= cacheBudget) pieceCache.clear();
|
|
89
|
+
const piece = pieceAt(tokenizer, tokenId);
|
|
90
|
+
pieceCache.set(tokenId, piece);
|
|
91
|
+
return piece;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Per-line tab count resets on each emitted newline. `linesEmitted`
|
|
95
|
+
// counts complete lines so the mask can permit EOS once at least one
|
|
96
|
+
// valid record has been produced.
|
|
97
|
+
let tabsOnCurrentLine = 0;
|
|
98
|
+
let linesEmitted = 0;
|
|
99
|
+
let lastGeneratedLen = 0;
|
|
100
|
+
|
|
101
|
+
function reflectEmittedTokens(tokenizer, generatedIds) {
|
|
102
|
+
for (let i = lastGeneratedLen; i < generatedIds.length; i += 1) {
|
|
103
|
+
const piece = cachedPiece(tokenizer, generatedIds[i]);
|
|
104
|
+
// Walk the piece char-by-char so multi-tab/newline pieces stay
|
|
105
|
+
// accurate (e.g., a token piece "\t\n" both completes a line
|
|
106
|
+
// AND resets the tab counter).
|
|
107
|
+
for (let c = 0; c < piece.length; c += 1) {
|
|
108
|
+
const ch = piece[c];
|
|
109
|
+
if (ch === "\n") {
|
|
110
|
+
if (tabsOnCurrentLine > 0) linesEmitted += 1;
|
|
111
|
+
tabsOnCurrentLine = 0;
|
|
112
|
+
} else if (ch === "\t") {
|
|
113
|
+
tabsOnCurrentLine += 1;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
lastGeneratedLen = generatedIds.length;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Walk a candidate piece's chars and check whether any intermediate
|
|
122
|
+
* state would violate the per-line tab budget. Returns true iff the
|
|
123
|
+
* piece is structurally valid given the current `tabsOnCurrentLine`.
|
|
124
|
+
*
|
|
125
|
+
* Intra-piece state evolution matters because BPE tokens occasionally
|
|
126
|
+
* span line boundaries (e.g., a piece `"\t\n"` ends one line AND
|
|
127
|
+
* starts the next with tabsOnCurrentLine=0). A naive check that just
|
|
128
|
+
* sums the piece's tabs/newlines would misjudge such tokens.
|
|
129
|
+
*/
|
|
130
|
+
function pieceFitsBudget(piece, startingTabs, tabsMax) {
|
|
131
|
+
let tabs = startingTabs;
|
|
132
|
+
let sawNewlineWithoutCompleteLine = false;
|
|
133
|
+
for (let c = 0; c < piece.length; c += 1) {
|
|
134
|
+
const ch = piece[c];
|
|
135
|
+
if (ch === "\n") {
|
|
136
|
+
if (tabs < tabsMax) sawNewlineWithoutCompleteLine = true;
|
|
137
|
+
tabs = 0;
|
|
138
|
+
} else if (ch === "\t") {
|
|
139
|
+
tabs += 1;
|
|
140
|
+
if (tabs > tabsMax) return false;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
if (sawNewlineWithoutCompleteLine) return false;
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return function logitMask(logits, context) {
|
|
148
|
+
const tokenizer = (opts.tokenizer ?? context?.tokenizer) || null;
|
|
149
|
+
if (!tokenizer) return;
|
|
150
|
+
reflectEmittedTokens(tokenizer, Array.isArray(context?.generatedIds) ? context.generatedIds : []);
|
|
151
|
+
|
|
152
|
+
const vocabSize = Math.min(logits.length, Number(context?.vocabSize || logits.length));
|
|
153
|
+
for (let id = 0; id < vocabSize; id += 1) {
|
|
154
|
+
if (logits[id] <= -Infinity) continue;
|
|
155
|
+
const piece = cachedPiece(tokenizer, id);
|
|
156
|
+
if (!piece) continue;
|
|
157
|
+
// Fast path: pieces with no tab/newline are always allowed —
|
|
158
|
+
// they're field content (snippet text, digits, decimal points).
|
|
159
|
+
// Avoids the per-char walk for the common case.
|
|
160
|
+
if (piece.indexOf("\t") < 0 && piece.indexOf("\n") < 0) continue;
|
|
161
|
+
if (!pieceFitsBudget(piece, tabsOnCurrentLine, tabsPerLineMax)) {
|
|
162
|
+
logits[id] = -Infinity;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
}
|
|
@@ -10,11 +10,6 @@
|
|
|
10
10
|
|
|
11
11
|
import type { AttentionConfig } from './types.js';
|
|
12
12
|
|
|
13
|
-
/**
|
|
14
|
-
* Required fields that must not be undefined on every AttnConfig instance.
|
|
15
|
-
*/
|
|
16
|
-
export declare const ATTN_CONFIG_REQUIRED_FIELDS: readonly string[];
|
|
17
|
-
|
|
18
13
|
/**
|
|
19
14
|
* Validate an AttnConfig object at construction time.
|
|
20
15
|
*
|
|
@@ -6,7 +6,7 @@ import { log } from '../../../../debug/index.js';
|
|
|
6
6
|
* entry points. Optional fields (skipInputNorm, tokenIds, kernelPath,
|
|
7
7
|
* disableRoPE) have defaults in the destructuring and are excluded.
|
|
8
8
|
*/
|
|
9
|
-
|
|
9
|
+
const ATTN_CONFIG_REQUIRED_FIELDS = Object.freeze([
|
|
10
10
|
'layerIdx',
|
|
11
11
|
'numTokens',
|
|
12
12
|
'isPrefill',
|
|
@@ -96,6 +96,7 @@ export interface ProjectAttentionQKVResult {
|
|
|
96
96
|
kTensor: Tensor;
|
|
97
97
|
vTensor: Tensor;
|
|
98
98
|
usedFusedQKV: boolean;
|
|
99
|
+
valueAliasesKey: boolean;
|
|
99
100
|
}
|
|
100
101
|
|
|
101
102
|
export function projectAttentionQKV(options: ProjectAttentionQKVOptions): Promise<ProjectAttentionQKVResult>;
|
|
@@ -115,6 +116,7 @@ export interface ApplyAttentionQKNormOptions {
|
|
|
115
116
|
releaseTemporary: (buffer: GPUBuffer) => void;
|
|
116
117
|
onQNormApplied?: ((tensor: Tensor) => Promise<void> | void) | null;
|
|
117
118
|
onKNormApplied?: ((tensor: Tensor) => Promise<void> | void) | null;
|
|
119
|
+
retainKInput?: boolean;
|
|
118
120
|
}
|
|
119
121
|
|
|
120
122
|
export function applyAttentionQKNorm(
|
|
@@ -157,6 +157,9 @@ async function projectSingleQkvTensor({
|
|
|
157
157
|
matmulDebug,
|
|
158
158
|
releaseTemporary,
|
|
159
159
|
executionPolicies = null,
|
|
160
|
+
fusedNormWeight = null,
|
|
161
|
+
fusedNormEps = null,
|
|
162
|
+
fusedNormOffset = false,
|
|
160
163
|
}) {
|
|
161
164
|
const runMatmulForMode = getMatmulRunner(recorder);
|
|
162
165
|
const layerWeight = layerWeights?.[weightKey];
|
|
@@ -178,6 +181,9 @@ async function projectSingleQkvTensor({
|
|
|
178
181
|
outputDtype: matmulOutputDtype,
|
|
179
182
|
matmulDebug,
|
|
180
183
|
executionPolicies,
|
|
184
|
+
normWeight: fusedNormWeight,
|
|
185
|
+
rmsNormEps: fusedNormEps,
|
|
186
|
+
rmsNormOffset: fusedNormOffset,
|
|
181
187
|
});
|
|
182
188
|
} finally {
|
|
183
189
|
releaseOwnedWeightBuffer(layerWeight, projBuffer, releaseTemporary);
|
|
@@ -269,6 +275,9 @@ async function projectQueryWithOptionalGate({
|
|
|
269
275
|
releaseTemporary,
|
|
270
276
|
attentionOutputGate,
|
|
271
277
|
executionPolicies = null,
|
|
278
|
+
fusedNormWeight = null,
|
|
279
|
+
fusedNormEps = null,
|
|
280
|
+
fusedNormOffset = false,
|
|
272
281
|
}) {
|
|
273
282
|
const qSize = numHeads * headDim;
|
|
274
283
|
const qWeight = layerWeights?.qProj;
|
|
@@ -296,6 +305,9 @@ async function projectQueryWithOptionalGate({
|
|
|
296
305
|
lora,
|
|
297
306
|
matmulDebug,
|
|
298
307
|
releaseTemporary,
|
|
308
|
+
fusedNormWeight,
|
|
309
|
+
fusedNormEps,
|
|
310
|
+
fusedNormOffset,
|
|
299
311
|
});
|
|
300
312
|
return { qTensor, qGateTensor: null };
|
|
301
313
|
}
|
|
@@ -433,6 +445,9 @@ export async function projectAttentionQKV({
|
|
|
433
445
|
sharedKTensor = null,
|
|
434
446
|
sharedVTensor = null,
|
|
435
447
|
executionPolicies = null,
|
|
448
|
+
fusedNormWeight = null,
|
|
449
|
+
fusedNormEps = null,
|
|
450
|
+
fusedNormOffset = false,
|
|
436
451
|
}) {
|
|
437
452
|
const runMatmulForMode = getMatmulRunner(recorder);
|
|
438
453
|
const runSplitForMode = getSplitRunner(recorder);
|
|
@@ -485,6 +500,12 @@ export async function projectAttentionQKV({
|
|
|
485
500
|
outputDtype: projectionOutputDtype,
|
|
486
501
|
matmulDebug,
|
|
487
502
|
executionPolicies,
|
|
503
|
+
// Forward fused-rmsnorm params so the combined QKV matmul runs the
|
|
504
|
+
// input_norm prologue internally, eliminating the standalone rmsnorm
|
|
505
|
+
// dispatch upstream for layers using the useFusedQKV path.
|
|
506
|
+
normWeight: fusedNormWeight,
|
|
507
|
+
rmsNormEps: fusedNormEps,
|
|
508
|
+
rmsNormOffset: fusedNormOffset,
|
|
488
509
|
});
|
|
489
510
|
const split = await runSplitForMode(qkvTensor, {
|
|
490
511
|
numTokens,
|
|
@@ -496,7 +517,14 @@ export async function projectAttentionQKV({
|
|
|
496
517
|
if (onFusedQKV) {
|
|
497
518
|
onFusedQKV({ qSize: qSizeFused, kSize: kSizeFused, vSize: vSizeFused, totalSize: qkvSizeTotal });
|
|
498
519
|
}
|
|
499
|
-
return {
|
|
520
|
+
return {
|
|
521
|
+
qTensor: split.Q,
|
|
522
|
+
qGateTensor: null,
|
|
523
|
+
kTensor: split.K,
|
|
524
|
+
vTensor: split.V,
|
|
525
|
+
usedFusedQKV: true,
|
|
526
|
+
valueAliasesKey: false,
|
|
527
|
+
};
|
|
500
528
|
} catch (error) {
|
|
501
529
|
if (qkvTensor) {
|
|
502
530
|
releaseTemporary(qkvTensor.buffer);
|
|
@@ -531,6 +559,9 @@ export async function projectAttentionQKV({
|
|
|
531
559
|
releaseTemporary,
|
|
532
560
|
attentionOutputGate,
|
|
533
561
|
executionPolicies,
|
|
562
|
+
fusedNormWeight,
|
|
563
|
+
fusedNormEps,
|
|
564
|
+
fusedNormOffset,
|
|
534
565
|
}));
|
|
535
566
|
|
|
536
567
|
if (reuseSharedKV) {
|
|
@@ -540,6 +571,7 @@ export async function projectAttentionQKV({
|
|
|
540
571
|
kTensor: sharedKTensor,
|
|
541
572
|
vTensor: sharedVTensor,
|
|
542
573
|
usedFusedQKV: false,
|
|
574
|
+
valueAliasesKey: false,
|
|
543
575
|
};
|
|
544
576
|
}
|
|
545
577
|
|
|
@@ -562,30 +594,42 @@ export async function projectAttentionQKV({
|
|
|
562
594
|
matmulDebug,
|
|
563
595
|
releaseTemporary,
|
|
564
596
|
executionPolicies,
|
|
597
|
+
fusedNormWeight,
|
|
598
|
+
fusedNormEps,
|
|
599
|
+
fusedNormOffset,
|
|
565
600
|
});
|
|
566
601
|
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
602
|
+
let valueAliasesKey = false;
|
|
603
|
+
if (layerWeights.vProj) {
|
|
604
|
+
vTensor = await projectSingleQkvTensor({
|
|
605
|
+
recorder,
|
|
606
|
+
normed: projectionInput,
|
|
607
|
+
layerWeights,
|
|
608
|
+
weightKey: 'vProj',
|
|
609
|
+
role: 'v_proj',
|
|
610
|
+
outputSize: numKVHeads * headDim,
|
|
611
|
+
outputLabel: 'V',
|
|
612
|
+
loraKey: 'v_proj',
|
|
613
|
+
numTokens,
|
|
614
|
+
hiddenSize,
|
|
615
|
+
layerIdx,
|
|
616
|
+
kernelPath,
|
|
617
|
+
matmulOutputDtype: projectionOutputDtype,
|
|
618
|
+
getWeightBuffer,
|
|
619
|
+
lora,
|
|
620
|
+
matmulDebug,
|
|
621
|
+
releaseTemporary,
|
|
622
|
+
executionPolicies,
|
|
623
|
+
fusedNormWeight,
|
|
624
|
+
fusedNormEps,
|
|
625
|
+
fusedNormOffset,
|
|
626
|
+
});
|
|
627
|
+
} else {
|
|
628
|
+
vTensor = kTensor;
|
|
629
|
+
valueAliasesKey = true;
|
|
630
|
+
}
|
|
587
631
|
|
|
588
|
-
return { qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV: false };
|
|
632
|
+
return { qTensor, qGateTensor, kTensor, vTensor, usedFusedQKV: false, valueAliasesKey };
|
|
589
633
|
} catch (error) {
|
|
590
634
|
for (const tensor of [qTensor, qGateTensor]) {
|
|
591
635
|
if (tensor?.buffer) {
|
|
@@ -645,6 +689,7 @@ export async function applyAttentionQKNorm({
|
|
|
645
689
|
onQNormApplied = null,
|
|
646
690
|
onKNormApplied = null,
|
|
647
691
|
skipKNorm = false,
|
|
692
|
+
retainKInput = false,
|
|
648
693
|
}) {
|
|
649
694
|
const runRmsNormForMode = getRmsNormRunner(recorder);
|
|
650
695
|
let nextQ = qTensor;
|
|
@@ -683,7 +728,9 @@ export async function applyAttentionQKNorm({
|
|
|
683
728
|
hiddenSize: headDim,
|
|
684
729
|
rmsNormWeightOffset,
|
|
685
730
|
});
|
|
686
|
-
|
|
731
|
+
if (!retainKInput) {
|
|
732
|
+
releaseTemporary(nextK.buffer);
|
|
733
|
+
}
|
|
687
734
|
nextK = kNormedTensor;
|
|
688
735
|
if (onKNormApplied) {
|
|
689
736
|
await onKNormApplied(nextK);
|