doppler-gpu 0.2.2 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +20 -8
- package/assets/doppler.svg +1601 -0
- package/package.json +82 -33
- package/src/cli/cli-model-resolution.js +275 -0
- package/src/cli/cli-output.js +453 -0
- package/src/cli/doppler-cli.js +817 -709
- package/src/cli/doppler-quickstart.d.ts +7 -23
- package/src/cli/doppler-quickstart.js +20 -4
- package/src/cli/doppler-serve.js +1 -1
- package/src/cli/runtime-profiles.d.ts +40 -0
- package/src/cli/runtime-profiles.js +225 -0
- package/src/client/doppler-api.js +1 -6
- package/src/client/doppler-registry.d.ts +12 -0
- package/src/client/doppler-registry.js +27 -0
- package/src/client/doppler-registry.json +79 -5
- package/src/client/failure-taxonomy.js +139 -0
- package/src/client/fault-injection.js +30 -0
- package/src/client/provider.d.ts +151 -0
- package/src/client/provider.js +374 -0
- package/src/client/receipt.js +156 -0
- package/src/client/runtime/index.js +56 -4
- package/src/client/runtime/model-manager.d.ts +0 -14
- package/src/client/runtime/model-manager.js +2 -42
- package/src/client/runtime/model-session.js +33 -0
- package/src/client/runtime/model-source.d.ts +13 -1
- package/src/client/runtime/model-source.js +128 -2
- package/src/client/runtime/node-quickstart-cache.d.ts +22 -0
- package/src/client/runtime/node-quickstart-cache.js +353 -0
- package/src/client/runtime/source-runtime.js +1 -10
- package/src/client/runtime/types.d.ts +0 -60
- package/src/client/wrap-pipeline-handle.d.ts +20 -0
- package/src/client/wrap-pipeline-handle.js +84 -0
- package/src/config/README.md +3 -1
- package/src/config/conversion/README.md +23 -157
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +19 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +16 -9
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +16 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +24 -9
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +719 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +715 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +589 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +566 -0
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +368 -62
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +212 -55
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +216 -57
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +229 -57
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +17 -10
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +75 -33
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +91 -59
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +517 -0
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +511 -0
- package/src/config/execution-contract-check.js +6 -5
- package/src/config/kernel-path-loader.d.ts +8 -2
- package/src/config/kernels/kernel-ref-digests.js +125 -122
- package/src/config/kernels/registry.d.ts +0 -35
- package/src/config/kernels/registry.js +5 -9
- package/src/config/kernels/registry.json +615 -368
- package/src/config/merge-helpers.d.ts +1 -1
- package/src/config/merge-helpers.js +1 -1
- package/src/config/merge.js +35 -7
- package/src/config/param-categories.d.ts +0 -9
- package/src/config/param-categories.js +1 -5
- package/src/config/param-validator.d.ts +2 -0
- package/src/config/param-validator.js +16 -0
- package/src/config/platforms/loader.d.ts +0 -27
- package/src/config/platforms/loader.js +0 -30
- package/src/config/required-inference-fields-contract-check.js +1 -7
- package/src/config/runtime/README.md +28 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +2 -2
- package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +85 -0
- package/src/config/runtime/platform/metal-apple-q4k.json +19 -3
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +65 -0
- package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +41 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +44 -0
- package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +19 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +31 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +34 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +64 -0
- package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +47 -0
- package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +35 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +16 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +62 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/bridge.schema.d.ts +16 -0
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/converter.schema.d.ts +5 -0
- package/src/config/schema/converter.schema.js +3 -0
- package/src/config/schema/debug.schema.d.ts +22 -1
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/distill-training.schema.d.ts +0 -1
- package/src/config/schema/distill-training.schema.js +1 -1
- package/src/config/schema/doppler.schema.d.ts +21 -3
- package/src/config/schema/doppler.schema.js +116 -9
- package/src/config/schema/emulation.schema.d.ts +3 -0
- package/src/config/schema/emulation.schema.js +16 -16
- package/src/config/schema/execution-v1.schema.d.ts +12 -1
- package/src/config/schema/execution-v1.schema.js +7 -1
- package/src/config/schema/index.d.ts +54 -1
- package/src/config/schema/index.js +15 -0
- package/src/config/schema/inference.schema.d.ts +3 -3
- package/src/config/schema/kernel-path.schema.d.ts +6 -27
- package/src/config/schema/kernel-thresholds.schema.d.ts +0 -2
- package/src/config/schema/kernel-thresholds.schema.js +3 -7
- package/src/config/schema/kvcache.schema.js +0 -9
- package/src/config/schema/loading.schema.d.ts +12 -0
- package/src/config/schema/loading.schema.js +4 -0
- package/src/config/schema/manifest.schema.d.ts +73 -1
- package/src/config/schema/manifest.schema.js +48 -2
- package/src/config/schema/moe.schema.js +0 -8
- package/src/config/schema/program-bundle.schema.d.ts +233 -0
- package/src/config/schema/program-bundle.schema.js +393 -0
- package/src/config/schema/ul-training.schema.d.ts +0 -1
- package/src/config/schema/ul-training.schema.js +1 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +36 -15
- package/src/config/supported-operations.d.ts +14 -0
- package/src/config/supported-operations.js +34 -0
- package/src/config/supported-operations.json +178 -0
- package/src/config/training-defaults.js +1 -7
- package/src/config/transforms/capability-transform-resolver.js +3 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +55 -0
- package/src/config/transforms/execution-graph-transforms.js +693 -29
- package/src/converter/conversion-plan.js +13 -10
- package/src/converter/core.d.ts +57 -0
- package/src/converter/core.js +582 -13
- package/src/converter/parsers/diffusion.js +1 -33
- package/src/converter/quantization-info.d.ts +5 -4
- package/src/converter/quantization-info.js +39 -3
- package/src/converter/quantizer.d.ts +19 -0
- package/src/converter/quantizer.js +81 -4
- package/src/debug/config.d.ts +0 -4
- package/src/debug/config.js +0 -17
- package/src/debug/index.d.ts +28 -30
- package/src/debug/index.js +0 -18
- package/src/debug/stats.d.ts +0 -8
- package/src/debug/stats.js +3 -3
- package/src/debug/tensor.d.ts +9 -2
- package/src/debug/tensor.js +11 -4
- package/src/debug/trace.js +2 -15
- package/src/errors/doppler-error.js +20 -0
- package/src/experimental/browser/quantization.d.ts +5 -0
- package/src/experimental/distribution/shard-delivery.js +179 -23
- package/src/experimental/training/artifacts.js +1 -14
- package/src/experimental/training/checkpoint.js +1 -14
- package/src/experimental/training/datasets/token-batch.d.ts +1 -1
- package/src/experimental/training/datasets/token-batch.js +2 -2
- package/src/experimental/training/distillation/dataset.js +2 -15
- package/src/experimental/training/distillation/eval.js +3 -3
- package/src/experimental/training/distillation/runtime.js +3 -16
- package/src/experimental/training/distillation/student-fixture.js +6 -6
- package/src/experimental/training/lora-pipeline.js +1 -14
- package/src/experimental/training/objectives/base.d.ts +2 -2
- package/src/experimental/training/objectives/cross_entropy.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.d.ts +2 -2
- package/src/experimental/training/objectives/distill_kd.js +2 -2
- package/src/experimental/training/objectives/distill_triplet.d.ts +2 -2
- package/src/experimental/training/objectives/distill_triplet.js +2 -2
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +2 -2
- package/src/experimental/training/objectives/ul_stage2_base.js +1 -1
- package/src/experimental/training/operator-artifacts.js +1 -14
- package/src/experimental/training/runner.js +1 -14
- package/src/experimental/training/workload-packs/registry.json +6 -6
- package/src/experimental/training/workloads.js +1 -1
- package/src/formats/gguf/types.d.ts +0 -63
- package/src/formats/gguf/types.js +58 -38
- package/src/formats/litert/types.d.ts +0 -13
- package/src/formats/litert/types.js +2 -2
- package/src/formats/rdrr/distributed/index.d.ts +34 -0
- package/src/formats/rdrr/distributed/index.js +3 -0
- package/src/formats/rdrr/distributed/parsing.d.ts +6 -0
- package/src/formats/rdrr/distributed/parsing.js +23 -0
- package/src/formats/rdrr/distributed/validation.d.ts +23 -0
- package/src/formats/rdrr/distributed/validation.js +131 -0
- package/src/formats/rdrr/groups.d.ts +1 -13
- package/src/formats/rdrr/groups.js +0 -34
- package/src/formats/rdrr/index.d.ts +9 -0
- package/src/formats/rdrr/index.js +9 -0
- package/src/formats/rdrr/integrity.d.ts +51 -0
- package/src/formats/rdrr/integrity.js +200 -0
- package/src/formats/rdrr/lowerings.d.ts +28 -0
- package/src/formats/rdrr/lowerings.js +112 -0
- package/src/formats/rdrr/manifest.d.ts +0 -14
- package/src/formats/rdrr/manifest.js +5 -58
- package/src/formats/rdrr/merkle.d.ts +24 -0
- package/src/formats/rdrr/merkle.js +118 -0
- package/src/formats/rdrr/parsing.js +4 -0
- package/src/formats/rdrr/slice-compiler.d.ts +29 -0
- package/src/formats/rdrr/slice-compiler.js +188 -0
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/storage-descriptor.d.ts +36 -0
- package/src/formats/rdrr/storage-descriptor.js +186 -0
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -9
- package/src/formats/rdrr/tensor-config-validator.js +0 -30
- package/src/formats/rdrr/types.d.ts +115 -1
- package/src/formats/rdrr/validation.js +245 -0
- package/src/formats/tflite/types.d.ts +8 -31
- package/src/formats/tflite/types.js +50 -5
- package/src/gpu/command-recorder.d.ts +9 -21
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/device.d.ts +17 -3
- package/src/gpu/device.js +20 -4
- package/src/gpu/kernel-runtime.d.ts +8 -15
- package/src/gpu/kernel-selection-log.d.ts +0 -8
- package/src/gpu/kernel-selection-log.js +0 -15
- package/src/gpu/kernel-tuner/cache.js +0 -3
- package/src/gpu/kernels/README.md +50 -18
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +25 -0
- package/src/gpu/kernels/attention.d.ts +28 -19
- package/src/gpu/kernels/attention.js +443 -9
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_contiguous_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_f16.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_online_f16.wgsl +32 -29
- package/src/gpu/kernels/attention_decode_online_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_paged_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_decode_tiered_turboquant_prod_f16kv.wgsl +0 -1
- package/src/gpu/kernels/attention_f16.wgsl +88 -30
- package/src/gpu/kernels/attention_f16kv.wgsl +45 -15
- package/src/gpu/kernels/attention_head512_f16.wgsl +270 -0
- package/src/gpu/kernels/attention_head512_f16kv.wgsl +271 -0
- package/src/gpu/kernels/attention_prefill_flash_head256_f16kv.wgsl +306 -0
- package/src/gpu/kernels/attention_prefill_flash_reduce.wgsl +101 -0
- package/src/gpu/kernels/attention_small_f16.wgsl +22 -20
- package/src/gpu/kernels/attention_small_f16kv.wgsl +1 -1
- package/src/gpu/kernels/attention_streaming_f16.wgsl +39 -16
- package/src/gpu/kernels/attention_streaming_f16kv.wgsl +22 -3
- package/src/gpu/kernels/backward/index.d.ts +6 -0
- package/src/gpu/kernels/backward/layernorm_backward.wgsl +0 -1
- package/src/gpu/kernels/backward/utils.d.ts +35 -0
- package/src/gpu/kernels/bf16_to_f32.wgsl +0 -16
- package/src/gpu/kernels/bias_add_f16.wgsl +1 -1
- package/src/gpu/kernels/cast_f32_to_f16.wgsl +0 -1
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +19 -215
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +8 -82
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +92 -59
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +2 -85
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +27 -24
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +1 -1
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +249 -179
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +133 -235
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +119 -90
- package/src/gpu/kernels/codegen/wgsl-variants.js +142 -8
- package/src/gpu/kernels/constants.d.ts +9 -48
- package/src/gpu/kernels/constants.js +0 -23
- package/src/gpu/kernels/conv2d.d.ts +0 -4
- package/src/gpu/kernels/conv2d_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant.d.ts +0 -16
- package/src/gpu/kernels/dequant.js +1 -71
- package/src/gpu/kernels/dequant_mxfp4_expert_f16.wgsl +1 -1
- package/src/gpu/kernels/dequant_shared.wgsl +1 -55
- package/src/gpu/kernels/dispatch.d.ts +12 -97
- package/src/gpu/kernels/dispatch.js +0 -117
- package/src/gpu/kernels/energy_eval_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_grad_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_reduce_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_quintel_update_f16.wgsl +1 -1
- package/src/gpu/kernels/energy_update_f16.wgsl +1 -1
- package/src/gpu/kernels/fused-gate-up-gelu.js +121 -0
- package/src/gpu/kernels/fused_ffn_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +0 -19
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +0 -18
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +124 -0
- package/src/gpu/kernels/fused_matmul_q4.wgsl +23 -439
- package/src/gpu/kernels/fused_matmul_q4_batched_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_batched_f16acc_f16a.wgsl +158 -0
- package/src/gpu/kernels/fused_matmul_q4_f16a.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16.wgsl +0 -1
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a.wgsl +32 -33
- package/src/gpu/kernels/fused_matmul_q4_multicol_f16a_f32acc.wgsl +209 -0
- package/src/gpu/kernels/fused_matmul_q4_prefill_tiled_f16.wgsl +261 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile.wgsl +164 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16.wgsl +198 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_f16a.wgsl +161 -0
- package/src/gpu/kernels/fused_matmul_q4_widetile_residual.wgsl +173 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm_f16.wgsl +1 -1
- package/src/gpu/kernels/fused_rmsnorm_q4_widetile.wgsl +271 -0
- package/src/gpu/kernels/fused_swiglu_f16.wgsl +1 -1
- package/src/gpu/kernels/gather.d.ts +0 -7
- package/src/gpu/kernels/gather_f16.wgsl +1 -1
- package/src/gpu/kernels/gelu_f16.wgsl +14 -15
- package/src/gpu/kernels/groupnorm.d.ts +0 -4
- package/src/gpu/kernels/groupnorm_apply_f16.wgsl +1 -1
- package/src/gpu/kernels/groupnorm_stats_f16.wgsl +1 -1
- package/src/gpu/kernels/index.d.ts +13 -5
- package/src/gpu/kernels/index.js +9 -4
- package/src/gpu/kernels/kv_quantize_turboquant.wgsl +0 -2
- package/src/gpu/kernels/kv_quantize_turboquant_prod.wgsl +0 -2
- package/src/gpu/kernels/layernorm.d.ts +0 -4
- package/src/gpu/kernels/layernorm_f16.wgsl +1 -1
- package/src/gpu/kernels/{sana_linear_attention.d.ts → linear_attention.d.ts} +5 -5
- package/src/gpu/kernels/{sana_linear_attention.js → linear_attention.js} +16 -16
- package/src/gpu/kernels/logit-merge.d.ts +0 -24
- package/src/gpu/kernels/logit-merge.js +1 -17
- package/src/gpu/kernels/matmul-dispatch.js +7 -1
- package/src/gpu/kernels/matmul-selection.js +68 -1
- package/src/gpu/kernels/matmul.d.ts +0 -28
- package/src/gpu/kernels/matmul.js +101 -7
- package/src/gpu/kernels/matmul_f16_tiled.wgsl +33 -34
- package/src/gpu/kernels/matmul_gemv_residual_f16.wgsl +1 -1
- package/src/gpu/kernels/matmul_gemv_subgroup.wgsl +0 -1
- package/src/gpu/kernels/matmul_gemv_subgroup_f16a.wgsl +24 -25
- package/src/gpu/kernels/modulate_f16.wgsl +1 -1
- package/src/gpu/kernels/moe.d.ts +0 -56
- package/src/gpu/kernels/moe.js +0 -15
- package/src/gpu/kernels/moe_gather_f16.wgsl +1 -1
- package/src/gpu/kernels/pixel_shuffle_f16.wgsl +1 -1
- package/src/gpu/kernels/residual.d.ts +0 -12
- package/src/gpu/kernels/residual_f16.wgsl +2 -2
- package/src/gpu/kernels/residual_f16_vec4.wgsl +5 -5
- package/src/gpu/kernels/rmsnorm.d.ts +0 -8
- package/src/gpu/kernels/rmsnorm.js +17 -1
- package/src/gpu/kernels/rmsnorm.wgsl +5 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +65 -39
- package/src/gpu/kernels/rope.d.ts +1 -5
- package/src/gpu/kernels/rope.js +12 -2
- package/src/gpu/kernels/rope.wgsl +33 -19
- package/src/gpu/kernels/rope_f16.wgsl +39 -25
- package/src/gpu/kernels/sample.d.ts +0 -5
- package/src/gpu/kernels/sample.wgsl +7 -6
- package/src/gpu/kernels/sample_f16.wgsl +28 -30
- package/src/gpu/kernels/scatter_add_dynamic_f16.wgsl +1 -1
- package/src/gpu/kernels/shader-cache.d.ts +13 -0
- package/src/gpu/kernels/shader-cache.js +27 -0
- package/src/gpu/kernels/silu_f16.wgsl +1 -1
- package/src/gpu/kernels/softmax.wgsl +4 -144
- package/src/gpu/kernels/split_qg_f16.wgsl +1 -1
- package/src/gpu/kernels/split_qkv_f16.wgsl +1 -1
- package/src/gpu/kernels/topk_f16.wgsl +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +11 -40
- package/src/gpu/kernels/uniform-utils.d.ts +18 -0
- package/src/gpu/kernels/upsample2d.d.ts +0 -4
- package/src/gpu/kernels/upsample2d_f16.wgsl +1 -1
- package/src/gpu/kernels/utils.d.ts +28 -0
- package/src/gpu/kernels/utils.js +41 -0
- package/src/gpu/perf-guards.d.ts +0 -12
- package/src/gpu/perf-guards.js +0 -62
- package/src/gpu/profiler.d.ts +0 -21
- package/src/gpu/profiler.js +0 -34
- package/src/gpu/submit-tracker.d.ts +0 -57
- package/src/gpu/submit-tracker.js +3 -62
- package/src/gpu/tensor.d.ts +0 -23
- package/src/gpu/tensor.js +0 -41
- package/src/gpu/uniform-cache.d.ts +0 -11
- package/src/gpu/uniform-cache.js +0 -23
- package/src/index.d.ts +1 -0
- package/src/index.js +1 -0
- package/src/inference/browser-harness-model-helpers.d.ts +0 -9
- package/src/inference/browser-harness-model-helpers.js +22 -13
- package/src/inference/browser-harness-runtime-helpers.d.ts +0 -5
- package/src/inference/browser-harness-runtime-helpers.js +2 -2
- package/src/inference/browser-harness-suite-helpers.d.ts +4 -1
- package/src/inference/browser-harness-suite-helpers.js +87 -1
- package/src/inference/browser-harness-text-helpers.d.ts +13 -0
- package/src/inference/browser-harness-text-helpers.js +267 -18
- package/src/inference/browser-harness.d.ts +0 -29
- package/src/inference/browser-harness.js +271 -71
- package/src/inference/kv-cache/base.d.ts +6 -0
- package/src/inference/kv-cache/base.js +16 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +1 -0
- package/src/inference/kv-cache/mixed-geometry.js +12 -2
- package/src/inference/kv-cache/quantized.js +2 -0
- package/src/inference/kv-cache/tiered.js +7 -0
- package/src/inference/kv-cache/types.d.ts +2 -0
- package/src/inference/multi-model-network.d.ts +1 -1
- package/src/inference/multi-model-network.js +1 -2
- package/src/inference/network-evolution.d.ts +0 -12
- package/src/inference/network-evolution.js +2 -2
- package/src/inference/pipelines/context.js +21 -2
- package/src/inference/pipelines/diffusion/init.js +2 -2
- package/src/inference/pipelines/diffusion/pipeline.js +41 -96
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -5
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +0 -283
- package/src/inference/pipelines/diffusion/vae.js +4 -4
- package/src/inference/pipelines/energy/quintel.d.ts +15 -0
- package/src/inference/pipelines/structured/index.d.ts +2 -0
- package/src/inference/pipelines/structured/index.js +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +143 -0
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +166 -0
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -5
- package/src/inference/pipelines/text/attention/attn-config.js +1 -1
- package/src/inference/pipelines/text/attention/projections.d.ts +2 -0
- package/src/inference/pipelines/text/attention/projections.js +70 -23
- package/src/inference/pipelines/text/attention/record.js +118 -5
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +8 -0
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.js +19 -0
- package/src/inference/pipelines/text/attention/run.js +97 -6
- package/src/inference/pipelines/text/attention/types.d.ts +2 -0
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/chat-format.d.ts +0 -10
- package/src/inference/pipelines/text/chat-format.js +1 -8
- package/src/inference/pipelines/text/config.d.ts +3 -25
- package/src/inference/pipelines/text/config.js +105 -3
- package/src/inference/pipelines/text/debug-utils/utils.js +2 -23
- package/src/inference/pipelines/text/drift-policy.js +1 -1
- package/src/inference/pipelines/text/embed.d.ts +23 -27
- package/src/inference/pipelines/text/embed.js +10 -43
- package/src/inference/pipelines/text/execution-plan.d.ts +5 -3
- package/src/inference/pipelines/text/execution-plan.js +15 -2
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -4
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -11
- package/src/inference/pipelines/text/execution-v1.d.ts +18 -0
- package/src/inference/pipelines/text/execution-v1.js +94 -25
- package/src/inference/pipelines/text/ffn/dense.js +180 -3
- package/src/inference/pipelines/text/ffn/standard.js +40 -13
- package/src/inference/pipelines/text/generator-decode-policy.js +243 -0
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -2
- package/src/inference/pipelines/text/generator-helpers.js +20 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.js +293 -0
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -7
- package/src/inference/pipelines/text/generator-runtime.js +8 -6
- package/src/inference/pipelines/text/generator-steps.d.ts +22 -7
- package/src/inference/pipelines/text/generator-steps.js +23 -9
- package/src/inference/pipelines/text/generator.d.ts +0 -6
- package/src/inference/pipelines/text/generator.js +254 -530
- package/src/inference/pipelines/text/init-chat-templates.js +86 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +201 -0
- package/src/inference/pipelines/text/init-rope.js +408 -0
- package/src/inference/pipelines/text/init.d.ts +8 -2
- package/src/inference/pipelines/text/init.js +190 -710
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -33
- package/src/inference/pipelines/text/kernel-trace.js +0 -33
- package/src/inference/pipelines/text/layer-plan-gpu.js +542 -0
- package/src/inference/pipelines/text/layer-plan.d.ts +0 -5
- package/src/inference/pipelines/text/layer-plan.js +1 -1
- package/src/inference/pipelines/text/layer.d.ts +70 -8
- package/src/inference/pipelines/text/layer.js +105 -537
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -1
- package/src/inference/pipelines/text/load-errors.d.ts +21 -0
- package/src/inference/pipelines/text/load-errors.js +59 -0
- package/src/inference/pipelines/text/logits/cpu.js +2 -20
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +148 -11
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/model-load.d.ts +5 -4
- package/src/inference/pipelines/text/model-load.js +74 -17
- package/src/inference/pipelines/text/ops.d.ts +13 -2
- package/src/inference/pipelines/text/ops.js +1 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +347 -59
- package/src/inference/pipelines/text/probes.d.ts +3 -6
- package/src/inference/pipelines/text/probes.js +198 -31
- package/src/inference/pipelines/text/sampling.d.ts +0 -5
- package/src/inference/pipelines/text/sampling.js +140 -0
- package/src/inference/pipelines/text/stage-names.js +12 -0
- package/src/inference/pipelines/text/state.js +2 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +409 -0
- package/src/inference/pipelines/text/types.d.ts +8 -78
- package/src/inference/pipelines/text/weights.d.ts +0 -50
- package/src/inference/pipelines/text/weights.js +2 -51
- package/src/inference/pipelines/text.d.ts +9 -1
- package/src/inference/pipelines/text.js +233 -29
- package/src/inference/pipelines/vision/index.js +0 -107
- package/src/inference/runtime-model.d.ts +0 -7
- package/src/inference/runtime-model.js +0 -23
- package/src/inference/test-harness.d.ts +6 -22
- package/src/inference/test-harness.js +97 -23
- package/src/inference/tokenizer.js +1 -10
- package/src/inference/tokenizers/sentencepiece.js +65 -15
- package/src/inference/tokenizers/types.d.ts +0 -21
- package/src/loader/doppler-loader.d.ts +3 -0
- package/src/loader/doppler-loader.js +90 -12
- package/src/loader/embedding-loader.js +30 -5
- package/src/loader/loader-types.d.ts +4 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/manifest-config.js +25 -1
- package/src/loader/memory-monitor.d.ts +17 -5
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/quantization-constants.d.ts +6 -0
- package/src/loader/shard-resolver.d.ts +1 -0
- package/src/loader/shard-resolver.js +15 -0
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +317 -34
- package/src/loader/tensors/tensor-loader.d.ts +2 -9
- package/src/loader/tensors/tensor-loader.js +84 -1
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/memory/address-table.d.ts +1 -9
- package/src/memory/buffer-pool.d.ts +0 -16
- package/src/memory/buffer-pool.js +1 -33
- package/src/memory/heap-manager.d.ts +18 -43
- package/src/memory/unified-detect.d.ts +19 -34
- package/src/models/embeddinggemma.d.ts +7 -0
- package/src/models/embeddinggemma.js +24 -0
- package/src/models/family.d.ts +24 -0
- package/src/models/family.js +25 -0
- package/src/models/gemma3.d.ts +7 -0
- package/src/models/gemma3.js +33 -0
- package/src/models/gemma4.d.ts +7 -0
- package/src/models/gemma4.js +33 -0
- package/src/models/qwen3.d.ts +7 -0
- package/src/models/qwen3.js +36 -0
- package/src/rules/converter/tokenizer.rules.json +5 -0
- package/src/rules/execution-rules-contract-check.js +72 -3
- package/src/rules/inference/capability-transforms.rules.json +87 -0
- package/src/rules/inference/execution.rules.json +11 -1
- package/src/rules/inference/ffn.rules.json +6 -2
- package/src/rules/loader/tensor-loader.rules.json +9 -0
- package/src/rules/rule-registry.js +53 -75
- package/src/storage/artifact-storage-context.d.ts +12 -3
- package/src/storage/artifact-storage-context.js +389 -98
- package/src/storage/backends/opfs-store.js +138 -88
- package/src/storage/blake3.js +9 -1
- package/src/storage/export.js +1 -5
- package/src/storage/normalize-model-id.js +3 -0
- package/src/storage/preflight.js +22 -0
- package/src/storage/quickstart-downloader.js +44 -56
- package/src/storage/reports.js +35 -11
- package/src/storage/shard-manager.d.ts +8 -2
- package/src/storage/shard-manager.js +176 -15
- package/src/storage/source-artifact-store.d.ts +0 -2
- package/src/storage/source-artifact-store.js +3 -9
- package/src/sw.js +0 -3
- package/src/tooling/browser-command-runner.js +39 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.d.ts +1 -0
- package/src/tooling/command-api-family-normalizers.js +49 -7
- package/src/tooling/command-api-helpers.d.ts +0 -2
- package/src/tooling/command-api-helpers.js +6 -2
- package/src/tooling/command-api.d.ts +20 -1
- package/src/tooling/command-api.js +10 -1
- package/src/tooling/command-envelope.d.ts +0 -1
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/command-runner-shared.d.ts +10 -0
- package/src/tooling/conversion-config-materializer.d.ts +0 -10
- package/src/tooling/conversion-config-materializer.js +1 -1
- package/src/tooling/diagnose-runner.js +1 -4
- package/src/tooling/hf-registry-utils.d.ts +2 -6
- package/src/tooling/hf-registry-utils.js +90 -19
- package/src/tooling/kernel-path-builder/index.js +1 -4
- package/src/tooling/kernel-path-builder/runtime-overlay.js +1 -3
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -10
- package/src/tooling/lean-execution-contract-runner.js +2 -2
- package/src/tooling/litert-package-runtime.d.ts +1 -1
- package/src/tooling/litert-package-runtime.js +373 -21
- package/src/tooling/node-browser-command-runner.d.ts +15 -0
- package/src/tooling/node-browser-command-runner.js +97 -13
- package/src/tooling/node-command-runner.js +36 -0
- package/src/tooling/node-convert-worker-pool.js +12 -0
- package/src/tooling/node-convert-worker.js +10 -1
- package/src/tooling/node-converter.js +90 -1
- package/src/tooling/node-source-runtime.js +257 -150
- package/src/tooling/opfs-cache.js +1 -4
- package/src/tooling/precision-replay-math.js +174 -0
- package/src/tooling/program-bundle-host.d.ts +16 -0
- package/src/tooling/program-bundle-host.js +20 -0
- package/src/tooling/program-bundle-parity.d.ts +28 -0
- package/src/tooling/program-bundle-parity.js +210 -0
- package/src/tooling/program-bundle.d.ts +61 -0
- package/src/tooling/program-bundle.js +1119 -0
- package/src/tooling/rdrr-integrity-refresh.d.ts +32 -0
- package/src/tooling/rdrr-integrity-refresh.js +271 -0
- package/src/tooling/reference-verify.js +191 -0
- package/src/tooling/source-artifact-adapter.d.ts +0 -20
- package/src/tooling/source-artifact-adapter.js +4 -4
- package/src/tooling/source-package-profiles.js +1 -7
- package/src/tooling/source-runtime-bundle.d.ts +27 -0
- package/src/tooling/source-runtime-bundle.js +56 -23
- package/src/tooling/source-runtime-converter-config.d.ts +0 -5
- package/src/tooling/source-runtime-converter-config.js +13 -9
- package/src/tooling/source-runtime-materializer.js +4 -8
- package/src/tooling-exports/device.d.ts +11 -0
- package/src/tooling-exports/device.js +14 -0
- package/src/tooling-exports/manifest.d.ts +8 -0
- package/src/tooling-exports/manifest.js +11 -0
- package/src/tooling-exports/storage.d.ts +25 -0
- package/src/tooling-exports/storage.js +29 -0
- package/src/tooling-exports/structured.d.ts +1 -0
- package/src/tooling-exports/structured.js +5 -0
- package/src/tooling-exports.d.ts +23 -0
- package/src/tooling-exports.js +17 -0
- package/src/tooling-exports.shared.d.ts +27 -30
- package/src/tooling-exports.shared.js +28 -29
- package/src/types/index.d.ts +0 -3
- package/src/types/index.js +0 -3
- package/src/utils/array-buffer.d.ts +4 -0
- package/src/utils/array-buffer.js +9 -0
- package/src/utils/canonical-hash.d.ts +7 -0
- package/src/utils/canonical-hash.js +74 -0
- package/src/utils/clone-json.d.ts +1 -0
- package/src/utils/clone-json.js +7 -0
- package/src/utils/encode-utf8.js +5 -0
- package/src/utils/index.d.ts +4 -1
- package/src/utils/index.js +4 -1
- package/src/utils/runtime-env.js +7 -0
- package/src/utils/sha256.d.ts +1 -0
- package/src/utils/sha256.js +18 -4
- package/src/utils/stable-sort-object.d.ts +1 -0
- package/src/utils/stable-sort-object.js +13 -0
- package/src/version.js +1 -1
- package/tools/convert-safetensors-node.js +1 -2
- package/src/bootstrap.d.ts +0 -1
- package/src/config/conversion/sana/sana-sprint-0.6b-f16.json +0 -247
- package/src/config/kernels/kernel-ref.d.ts +0 -17
- package/src/config/kernels/kernel-ref.js +0 -75
- package/src/config/runtime/kernels/dequant-f16-q4k.json +0 -13
- package/src/config/runtime/kernels/dequant-f32-q4k.json +0 -13
- package/src/config/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +0 -37
- package/src/config/runtime/kernels/fused-q4k.json +0 -13
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f16a.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +0 -33
- package/src/config/runtime/kernels/gemma2-q4k-fused-f32a.json +0 -33
- package/src/config/runtime/kernels/safe-q4k.json +0 -13
- package/src/config/runtime/overlays/debug-f32a.json +0 -3
- package/src/config/runtime/overlays/trace-config.json +0 -28
- package/src/config/runtime/profiles/gemma4-e2b-throughput.json +0 -39
- package/src/debug/perf.d.ts +0 -33
- package/src/debug/perf.js +0 -51
- package/src/errors/index.d.ts +0 -1
- package/src/errors/index.js +0 -1
- package/src/formats/gguf/index.d.ts +0 -8
- package/src/formats/gguf/index.js +0 -4
- package/src/formats/litert/index.d.ts +0 -1
- package/src/formats/litert/index.js +0 -1
- package/src/formats/safetensors/index.d.ts +0 -8
- package/src/formats/safetensors/index.js +0 -4
- package/src/formats/tflite/index.d.ts +0 -1
- package/src/formats/tflite/index.js +0 -1
- package/src/gpu/kernels/attention_decode_optimized.wgsl +0 -445
- package/src/gpu/kernels/dequant_mxfp4_vec4.wgsl +0 -116
- package/src/gpu/kernels/dequant_q8_0.wgsl +0 -98
- package/src/gpu/kernels/dequant_shared_vec4.wgsl +0 -160
- package/src/gpu/kernels/moe_gather_vec4.wgsl +0 -74
- package/src/inference/functiongemma.d.ts +0 -15
- package/src/inference/functiongemma.js +0 -1
- package/src/inference/pipelines/diffusion/sana-transformer.d.ts +0 -53
- package/src/inference/pipelines/diffusion/sana-transformer.js +0 -738
- package/src/rules/kernels/kernel-validator.d.ts +0 -24
- package/src/rules/kernels/kernel-validator.js +0 -160
- package/src/sw.d.ts +0 -1
- package/src/tooling/bench-runner.js +0 -64
- package/src/tooling/distill-runner.js +0 -56
- package/src/tooling/lora-runner.js +0 -37
- package/src/tooling/verify-runner.js +0 -63
- package/src/types/gpu.d.ts +0 -185
- package/src/types/gpu.js +0 -5
- package/src/types/inference.d.ts +0 -197
- package/src/types/inference.js +0 -5
- package/src/types/model.d.ts +0 -130
- package/src/types/model.js +0 -5
- package/src/utils/format-bytes.js +0 -20
- /package/src/config/{runtime/diagnostics → diagnostics}/drift-policies.json +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply.wgsl → linear_attention_apply.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_apply_f16.wgsl → linear_attention_apply_f16.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary.wgsl → linear_attention_summary.wgsl} +0 -0
- /package/src/gpu/kernels/{sana_linear_attention_summary_f16.wgsl → linear_attention_summary_f16.wgsl} +0 -0
- /package/src/rules/kernels/{sana-linear-attention.rules.json → linear-attention.rules.json} +0 -0
|
@@ -12,14 +12,6 @@
|
|
|
12
12
|
* @module memory/address-table
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
/**
|
|
16
|
-
* Decoded address parts
|
|
17
|
-
*/
|
|
18
|
-
export interface DecodedAddress {
|
|
19
|
-
segmentIndex: number;
|
|
20
|
-
offset: number;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
15
|
/**
|
|
24
16
|
* Address range chunk for split operations
|
|
25
17
|
*/
|
|
@@ -63,7 +55,7 @@ export declare class AddressTable {
|
|
|
63
55
|
* Decode virtual address into segment index and offset
|
|
64
56
|
* @param virtualAddress - Virtual address to decode
|
|
65
57
|
*/
|
|
66
|
-
decode(virtualAddress: number):
|
|
58
|
+
decode(virtualAddress: number): { segmentIndex: number; offset: number };
|
|
67
59
|
|
|
68
60
|
/**
|
|
69
61
|
* Get the segment index from a virtual address
|
|
@@ -170,11 +170,6 @@ export declare class BufferPool {
|
|
|
170
170
|
*/
|
|
171
171
|
export function getBufferPool(): BufferPool;
|
|
172
172
|
|
|
173
|
-
/**
|
|
174
|
-
* Create a standalone buffer pool
|
|
175
|
-
*/
|
|
176
|
-
export function createBufferPool(debugMode?: boolean, schemaConfig?: BufferPoolConfigSchema): BufferPool;
|
|
177
|
-
|
|
178
173
|
/**
|
|
179
174
|
* Destroy the global buffer pool
|
|
180
175
|
*/
|
|
@@ -182,14 +177,12 @@ export function destroyBufferPool(): void;
|
|
|
182
177
|
|
|
183
178
|
// Convenience exports for common operations
|
|
184
179
|
export declare const createStagingBuffer: (size: number) => GPUBuffer;
|
|
185
|
-
export declare const createUploadBuffer: (size: number) => GPUBuffer;
|
|
186
180
|
export declare const createUniformBuffer: (size: number) => GPUBuffer;
|
|
187
181
|
export declare const acquireBuffer: (size: number, usage?: GPUBufferUsageFlags, label?: string) => GPUBuffer;
|
|
188
182
|
export declare const releaseBuffer: (buffer: GPUBuffer) => void;
|
|
189
183
|
export declare const discardBuffer: (buffer: GPUBuffer) => void;
|
|
190
184
|
export declare const isBufferActive: (buffer: GPUBuffer) => boolean;
|
|
191
185
|
export declare function markPersistentBuffer(buffer: GPUBuffer): void;
|
|
192
|
-
export declare function unmarkPersistentBuffer(buffer: GPUBuffer): void;
|
|
193
186
|
export declare function isPersistentBuffer(buffer: GPUBuffer): boolean;
|
|
194
187
|
export declare class PersistentBufferSet extends Set<GPUBuffer> {}
|
|
195
188
|
export declare const getBufferRequestedSize: (buffer: GPUBuffer) => number;
|
|
@@ -197,12 +190,3 @@ export declare const uploadData: (buffer: GPUBuffer, data: ArrayBuffer | ArrayBu
|
|
|
197
190
|
export declare const readBuffer: (buffer: GPUBuffer, size?: number) => Promise<ArrayBuffer>;
|
|
198
191
|
export declare const readBufferSlice: (buffer: GPUBuffer, offset: number, size: number) => Promise<ArrayBuffer>;
|
|
199
192
|
export declare const forceBufferPoolReclaim: (targetRatio?: number | null) => void;
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* Scoped buffer helper - automatically releases buffer when done
|
|
203
|
-
*/
|
|
204
|
-
export function withBuffer<T>(
|
|
205
|
-
size: number,
|
|
206
|
-
usage: GPUBufferUsageFlags,
|
|
207
|
-
fn: (buffer: GPUBuffer) => Promise<T>
|
|
208
|
-
): Promise<T>;
|
|
@@ -794,14 +794,6 @@ export function getBufferPool() {
|
|
|
794
794
|
}
|
|
795
795
|
|
|
796
796
|
|
|
797
|
-
export function createBufferPool(debugMode, schemaConfig) {
|
|
798
|
-
if (!schemaConfig) {
|
|
799
|
-
throw new Error('createBufferPool requires schemaConfig from runtime.shared.bufferPool.');
|
|
800
|
-
}
|
|
801
|
-
return new BufferPool(debugMode, schemaConfig);
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
|
|
805
797
|
export function destroyBufferPool() {
|
|
806
798
|
if (globalPool) {
|
|
807
799
|
globalPool.destroy();
|
|
@@ -814,8 +806,6 @@ export function destroyBufferPool() {
|
|
|
814
806
|
|
|
815
807
|
export const createStagingBuffer = (size) => getBufferPool().createStagingBuffer(size);
|
|
816
808
|
|
|
817
|
-
export const createUploadBuffer = (size) => getBufferPool().createUploadBuffer(size);
|
|
818
|
-
|
|
819
809
|
export const createUniformBuffer = (size) => getBufferPool().createUniformBuffer(size);
|
|
820
810
|
|
|
821
811
|
export const acquireBuffer = (size, usage, label) =>
|
|
@@ -823,15 +813,6 @@ export const acquireBuffer = (size, usage, label) =>
|
|
|
823
813
|
|
|
824
814
|
export const releaseBuffer = (buffer) => getBufferPool().release(buffer);
|
|
825
815
|
|
|
826
|
-
/**
|
|
827
|
-
* Safe release helper: no-op if buffer is null or undefined.
|
|
828
|
-
* Use in error/cleanup paths where the buffer may not have been acquired.
|
|
829
|
-
*/
|
|
830
|
-
export function safeRelease(buffer) {
|
|
831
|
-
if (buffer == null) return;
|
|
832
|
-
getBufferPool().release(buffer);
|
|
833
|
-
}
|
|
834
|
-
|
|
835
816
|
export const discardBuffer = (buffer) => getBufferPool().discard(buffer);
|
|
836
817
|
|
|
837
818
|
export const isBufferActive = (buffer) =>
|
|
@@ -843,7 +824,7 @@ export function markPersistentBuffer(buffer) {
|
|
|
843
824
|
}
|
|
844
825
|
}
|
|
845
826
|
|
|
846
|
-
|
|
827
|
+
function unmarkPersistentBuffer(buffer) {
|
|
847
828
|
if (buffer && typeof buffer === 'object') {
|
|
848
829
|
persistentBuffers.delete(buffer);
|
|
849
830
|
}
|
|
@@ -888,16 +869,3 @@ export const forceBufferPoolReclaim = (targetRatio) =>
|
|
|
888
869
|
getBufferPool().forceReclaim(targetRatio);
|
|
889
870
|
|
|
890
871
|
|
|
891
|
-
export async function withBuffer(
|
|
892
|
-
size,
|
|
893
|
-
usage,
|
|
894
|
-
fn
|
|
895
|
-
) {
|
|
896
|
-
const pool = getBufferPool();
|
|
897
|
-
const buffer = pool.acquire(size, usage);
|
|
898
|
-
try {
|
|
899
|
-
return await fn(buffer);
|
|
900
|
-
} finally {
|
|
901
|
-
pool.release(buffer);
|
|
902
|
-
}
|
|
903
|
-
}
|
|
@@ -9,47 +9,7 @@
|
|
|
9
9
|
* @module memory/heap-manager
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
-
import { MemoryStrategy
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Memory segment for segmented heap mode
|
|
16
|
-
*/
|
|
17
|
-
export interface MemorySegment {
|
|
18
|
-
index: number;
|
|
19
|
-
buffer: ArrayBuffer;
|
|
20
|
-
used: number;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
/**
|
|
24
|
-
* Allocation result
|
|
25
|
-
*/
|
|
26
|
-
export interface AllocationResult {
|
|
27
|
-
/** Virtual address for this allocation */
|
|
28
|
-
virtualAddress: number;
|
|
29
|
-
/** Allocated size in bytes */
|
|
30
|
-
size: number;
|
|
31
|
-
/** Direct view into the buffer */
|
|
32
|
-
view: Uint8Array;
|
|
33
|
-
/** Which strategy was used */
|
|
34
|
-
strategy: MemoryStrategy;
|
|
35
|
-
/** Segment index (segmented only) */
|
|
36
|
-
segmentIndex?: number;
|
|
37
|
-
/** Offset within segment (segmented only) */
|
|
38
|
-
segmentOffset?: number;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Heap manager statistics
|
|
43
|
-
*/
|
|
44
|
-
export interface HeapStats {
|
|
45
|
-
strategy: MemoryStrategy | null;
|
|
46
|
-
totalAllocated: number;
|
|
47
|
-
segmentCount: number;
|
|
48
|
-
memory64HeapSize: number;
|
|
49
|
-
allocated?: number;
|
|
50
|
-
limit?: number;
|
|
51
|
-
maxSize?: number;
|
|
52
|
-
}
|
|
12
|
+
import { MemoryStrategy } from './capability.js';
|
|
53
13
|
|
|
54
14
|
/**
|
|
55
15
|
* HeapManager - Unified interface for both memory strategies
|
|
@@ -64,7 +24,14 @@ export declare class HeapManager {
|
|
|
64
24
|
* Allocate buffer for model data
|
|
65
25
|
* @param size - Size in bytes
|
|
66
26
|
*/
|
|
67
|
-
allocate(size: number):
|
|
27
|
+
allocate(size: number): {
|
|
28
|
+
virtualAddress: number;
|
|
29
|
+
size: number;
|
|
30
|
+
view: Uint8Array;
|
|
31
|
+
strategy: MemoryStrategy;
|
|
32
|
+
segmentIndex?: number;
|
|
33
|
+
segmentOffset?: number;
|
|
34
|
+
};
|
|
68
35
|
|
|
69
36
|
/**
|
|
70
37
|
* Read data from virtual address
|
|
@@ -90,7 +57,15 @@ export declare class HeapManager {
|
|
|
90
57
|
/**
|
|
91
58
|
* Get memory stats
|
|
92
59
|
*/
|
|
93
|
-
getStats():
|
|
60
|
+
getStats(): {
|
|
61
|
+
strategy: MemoryStrategy | null;
|
|
62
|
+
totalAllocated: number;
|
|
63
|
+
segmentCount: number;
|
|
64
|
+
memory64HeapSize: number;
|
|
65
|
+
allocated?: number;
|
|
66
|
+
limit?: number;
|
|
67
|
+
maxSize?: number;
|
|
68
|
+
};
|
|
94
69
|
|
|
95
70
|
/**
|
|
96
71
|
* Free all memory (for model unload)
|
|
@@ -10,45 +10,30 @@
|
|
|
10
10
|
* @module memory/unified-detect
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
/**
|
|
14
|
-
* Apple Silicon detection result
|
|
15
|
-
*/
|
|
16
|
-
export interface AppleSiliconInfo {
|
|
17
|
-
isApple: boolean;
|
|
18
|
-
mSeriesGen?: number | null;
|
|
19
|
-
vendor?: string;
|
|
20
|
-
device?: string;
|
|
21
|
-
description?: string;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* AMD unified memory detection result
|
|
26
|
-
*/
|
|
27
|
-
export interface AMDUnifiedInfo {
|
|
28
|
-
isAMDUnified: boolean;
|
|
29
|
-
isStrix?: boolean;
|
|
30
|
-
vendor?: string;
|
|
31
|
-
device?: string;
|
|
32
|
-
description?: string;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* WebGPU buffer limit indicators
|
|
37
|
-
*/
|
|
38
|
-
export interface LimitIndicators {
|
|
39
|
-
largeBuffers: boolean;
|
|
40
|
-
maxBufferSize?: number;
|
|
41
|
-
maxStorageBufferBindingSize?: number;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
13
|
/**
|
|
45
14
|
* Unified memory detection result
|
|
46
15
|
*/
|
|
47
16
|
export interface UnifiedMemoryInfo {
|
|
48
17
|
isUnified: boolean;
|
|
49
|
-
apple?:
|
|
50
|
-
|
|
51
|
-
|
|
18
|
+
apple?: {
|
|
19
|
+
isApple: boolean;
|
|
20
|
+
mSeriesGen?: number | null;
|
|
21
|
+
vendor?: string;
|
|
22
|
+
device?: string;
|
|
23
|
+
description?: string;
|
|
24
|
+
};
|
|
25
|
+
amd?: {
|
|
26
|
+
isAMDUnified: boolean;
|
|
27
|
+
isStrix?: boolean;
|
|
28
|
+
vendor?: string;
|
|
29
|
+
device?: string;
|
|
30
|
+
description?: string;
|
|
31
|
+
};
|
|
32
|
+
limits?: {
|
|
33
|
+
largeBuffers: boolean;
|
|
34
|
+
maxBufferSize?: number;
|
|
35
|
+
maxStorageBufferBindingSize?: number;
|
|
36
|
+
};
|
|
52
37
|
estimatedMemoryGB?: number | null;
|
|
53
38
|
reason: string;
|
|
54
39
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { FamilyModelEntry } from './family.d.ts';
|
|
2
|
+
|
|
3
|
+
export declare const FAMILY_ID: string;
|
|
4
|
+
export declare const HF_REPO_ID: string;
|
|
5
|
+
export declare const KNOWN_MODELS: ReadonlyArray<FamilyModelEntry>;
|
|
6
|
+
export declare function resolveModel(modelId: string): FamilyModelEntry | null;
|
|
7
|
+
export declare function resolveHfBaseUrl(modelId: string, revision?: string): string | null;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Static metadata for the EmbeddingGemma family of RDRR artifacts.
|
|
2
|
+
|
|
3
|
+
import { createFamily } from './family.js';
|
|
4
|
+
|
|
5
|
+
const family = createFamily({
|
|
6
|
+
familyId: 'embeddinggemma',
|
|
7
|
+
hfRepoId: 'Clocksmith/rdrr',
|
|
8
|
+
knownModels: [
|
|
9
|
+
{
|
|
10
|
+
modelId: 'google-embeddinggemma-300m-q4k-ehf16-af32',
|
|
11
|
+
label: 'EmbeddingGemma 300M (Q4K)',
|
|
12
|
+
sourceModel: 'google/embeddinggemma-300m',
|
|
13
|
+
hfPath: 'models/google-embeddinggemma-300m-q4k-ehf16-af32',
|
|
14
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
15
|
+
modes: ['embedding'],
|
|
16
|
+
},
|
|
17
|
+
],
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
export const FAMILY_ID = family.FAMILY_ID;
|
|
21
|
+
export const HF_REPO_ID = family.HF_REPO_ID;
|
|
22
|
+
export const KNOWN_MODELS = family.KNOWN_MODELS;
|
|
23
|
+
export const resolveModel = family.resolveModel;
|
|
24
|
+
export const resolveHfBaseUrl = family.resolveHfBaseUrl;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export interface FamilyModelEntry {
|
|
2
|
+
readonly modelId: string;
|
|
3
|
+
readonly label: string;
|
|
4
|
+
readonly sourceModel: string;
|
|
5
|
+
readonly hfPath: string;
|
|
6
|
+
readonly defaultRuntimeProfile: string;
|
|
7
|
+
readonly modes: ReadonlyArray<'text' | 'vision' | 'embedding' | 'translate' | 'diffusion'>;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface FamilyModule {
|
|
11
|
+
readonly FAMILY_ID: string;
|
|
12
|
+
readonly HF_REPO_ID: string;
|
|
13
|
+
readonly KNOWN_MODELS: ReadonlyArray<FamilyModelEntry>;
|
|
14
|
+
resolveModel(modelId: string): FamilyModelEntry | null;
|
|
15
|
+
resolveHfBaseUrl(modelId: string, revision?: string): string | null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface CreateFamilyInput {
|
|
19
|
+
readonly familyId: string;
|
|
20
|
+
readonly hfRepoId: string;
|
|
21
|
+
readonly knownModels: ReadonlyArray<FamilyModelEntry>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export declare function createFamily(input: CreateFamilyInput): FamilyModule;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Shared factory for per-family pointer modules (qwen3, gemma3, gemma4,
|
|
2
|
+
// embeddinggemma). Centralizes the resolve helpers so each family file is
|
|
3
|
+
// pure data.
|
|
4
|
+
|
|
5
|
+
export function createFamily({ familyId, hfRepoId, knownModels }) {
|
|
6
|
+
const KNOWN_MODELS = Object.freeze(knownModels.map((m) => Object.freeze({ ...m, modes: Object.freeze([...m.modes]) })));
|
|
7
|
+
|
|
8
|
+
function resolveModel(modelId) {
|
|
9
|
+
return KNOWN_MODELS.find((m) => m.modelId === modelId) || null;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function resolveHfBaseUrl(modelId, revision = 'main') {
|
|
13
|
+
const entry = resolveModel(modelId);
|
|
14
|
+
if (!entry) return null;
|
|
15
|
+
return `https://huggingface.co/${hfRepoId}/resolve/${revision}/${entry.hfPath}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
return {
|
|
19
|
+
FAMILY_ID: familyId,
|
|
20
|
+
HF_REPO_ID: hfRepoId,
|
|
21
|
+
KNOWN_MODELS,
|
|
22
|
+
resolveModel,
|
|
23
|
+
resolveHfBaseUrl,
|
|
24
|
+
};
|
|
25
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { FamilyModelEntry } from './family.d.ts';
|
|
2
|
+
|
|
3
|
+
export declare const FAMILY_ID: string;
|
|
4
|
+
export declare const HF_REPO_ID: string;
|
|
5
|
+
export declare const KNOWN_MODELS: ReadonlyArray<FamilyModelEntry>;
|
|
6
|
+
export declare function resolveModel(modelId: string): FamilyModelEntry | null;
|
|
7
|
+
export declare function resolveHfBaseUrl(modelId: string, revision?: string): string | null;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Static metadata for the Gemma 3 family of RDRR artifacts.
|
|
2
|
+
// See qwen3.js for the contract.
|
|
3
|
+
|
|
4
|
+
import { createFamily } from './family.js';
|
|
5
|
+
|
|
6
|
+
const family = createFamily({
|
|
7
|
+
familyId: 'gemma3',
|
|
8
|
+
hfRepoId: 'Clocksmith/rdrr',
|
|
9
|
+
knownModels: [
|
|
10
|
+
{
|
|
11
|
+
modelId: 'gemma-3-270m-it-q4k-ehf16-af32',
|
|
12
|
+
label: 'Gemma 3 270M Instruct (Q4K/F32a)',
|
|
13
|
+
sourceModel: 'google/gemma-3-270m-it',
|
|
14
|
+
hfPath: 'models/gemma-3-270m-it-q4k-ehf16-af32',
|
|
15
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
16
|
+
modes: ['text', 'vision'],
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
modelId: 'gemma-3-1b-it-q4k-ehf16-af32',
|
|
20
|
+
label: 'Gemma 3 1B Instruct (Q4K/F32a)',
|
|
21
|
+
sourceModel: 'google/gemma-3-1b-it',
|
|
22
|
+
hfPath: 'models/gemma-3-1b-it-q4k-ehf16-af32',
|
|
23
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
24
|
+
modes: ['text', 'vision'],
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
export const FAMILY_ID = family.FAMILY_ID;
|
|
30
|
+
export const HF_REPO_ID = family.HF_REPO_ID;
|
|
31
|
+
export const KNOWN_MODELS = family.KNOWN_MODELS;
|
|
32
|
+
export const resolveModel = family.resolveModel;
|
|
33
|
+
export const resolveHfBaseUrl = family.resolveHfBaseUrl;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { FamilyModelEntry } from './family.d.ts';
|
|
2
|
+
|
|
3
|
+
export declare const FAMILY_ID: string;
|
|
4
|
+
export declare const HF_REPO_ID: string;
|
|
5
|
+
export declare const KNOWN_MODELS: ReadonlyArray<FamilyModelEntry>;
|
|
6
|
+
export declare function resolveModel(modelId: string): FamilyModelEntry | null;
|
|
7
|
+
export declare function resolveHfBaseUrl(modelId: string, revision?: string): string | null;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Static metadata for the Gemma 4 family of RDRR artifacts.
|
|
2
|
+
// See qwen3.js for the contract.
|
|
3
|
+
|
|
4
|
+
import { createFamily } from './family.js';
|
|
5
|
+
|
|
6
|
+
const family = createFamily({
|
|
7
|
+
familyId: 'gemma4',
|
|
8
|
+
hfRepoId: 'Clocksmith/rdrr',
|
|
9
|
+
knownModels: [
|
|
10
|
+
{
|
|
11
|
+
modelId: 'gemma-4-e2b-it-q4k-ehf16-af32',
|
|
12
|
+
label: 'Gemma 4 E2B Instruct (Q4K/F32a)',
|
|
13
|
+
sourceModel: 'google/gemma-4-e2b-it',
|
|
14
|
+
hfPath: 'models/gemma-4-e2b-it-q4k-ehf16-af32',
|
|
15
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
16
|
+
modes: ['text', 'vision'],
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
modelId: 'gemma-4-e2b-it-q4k-ehf16-af32-int4ple',
|
|
20
|
+
label: 'Gemma 4 E2B Instruct (Q4K/F32a/INT4 PLE)',
|
|
21
|
+
sourceModel: 'google/gemma-4-e2b-it',
|
|
22
|
+
hfPath: 'models/gemma-4-e2b-it-q4k-ehf16-af32-int4ple',
|
|
23
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
24
|
+
modes: ['text', 'vision'],
|
|
25
|
+
},
|
|
26
|
+
],
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
export const FAMILY_ID = family.FAMILY_ID;
|
|
30
|
+
export const HF_REPO_ID = family.HF_REPO_ID;
|
|
31
|
+
export const KNOWN_MODELS = family.KNOWN_MODELS;
|
|
32
|
+
export const resolveModel = family.resolveModel;
|
|
33
|
+
export const resolveHfBaseUrl = family.resolveHfBaseUrl;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { FamilyModelEntry } from './family.d.ts';
|
|
2
|
+
|
|
3
|
+
export declare const FAMILY_ID: string;
|
|
4
|
+
export declare const HF_REPO_ID: string;
|
|
5
|
+
export declare const KNOWN_MODELS: ReadonlyArray<FamilyModelEntry>;
|
|
6
|
+
export declare function resolveModel(modelId: string): FamilyModelEntry | null;
|
|
7
|
+
export declare function resolveHfBaseUrl(modelId: string, revision?: string): string | null;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
// Static metadata for the Qwen 3.5 family of RDRR artifacts.
|
|
2
|
+
// Importing this module does NOT load manifests or shards — it is a
|
|
3
|
+
// kilobyte-scale pointer surface for consumers that want a typed,
|
|
4
|
+
// tree-shakable way to reference "the Qwen family" without pulling
|
|
5
|
+
// the whole catalog or the runtime pipeline.
|
|
6
|
+
|
|
7
|
+
import { createFamily } from './family.js';
|
|
8
|
+
|
|
9
|
+
const family = createFamily({
|
|
10
|
+
familyId: 'qwen3',
|
|
11
|
+
hfRepoId: 'Clocksmith/rdrr',
|
|
12
|
+
knownModels: [
|
|
13
|
+
{
|
|
14
|
+
modelId: 'qwen-3-5-0-8b-q4k-ehaf16',
|
|
15
|
+
label: 'Qwen 3.5 0.8B (Q4K)',
|
|
16
|
+
sourceModel: 'Qwen/Qwen3.5-0.8B',
|
|
17
|
+
hfPath: 'models/qwen-3-5-0-8b-q4k-ehaf16',
|
|
18
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
19
|
+
modes: ['text', 'vision'],
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
modelId: 'qwen-3-5-2b-q4k-ehaf16',
|
|
23
|
+
label: 'Qwen 3.5 2B (Q4K)',
|
|
24
|
+
sourceModel: 'Qwen/Qwen3.5-2B',
|
|
25
|
+
hfPath: 'models/qwen-3-5-2b-q4k-ehaf16',
|
|
26
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
27
|
+
modes: ['text', 'vision'],
|
|
28
|
+
},
|
|
29
|
+
],
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
export const FAMILY_ID = family.FAMILY_ID;
|
|
33
|
+
export const HF_REPO_ID = family.HF_REPO_ID;
|
|
34
|
+
export const KNOWN_MODELS = family.KNOWN_MODELS;
|
|
35
|
+
export const resolveModel = family.resolveModel;
|
|
36
|
+
export const resolveHfBaseUrl = family.resolveHfBaseUrl;
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"type": [
|
|
3
|
+
{ "match": { "isLiteRTPieceList": true }, "value": "bpe" },
|
|
3
4
|
{ "match": { "hasMerges": true }, "value": "bpe" },
|
|
4
5
|
{ "match": { "hasScores": true }, "value": "unigram" },
|
|
5
6
|
{ "match": {}, "value": "bpe" }
|
|
7
|
+
],
|
|
8
|
+
"addDummyPrefix": [
|
|
9
|
+
{ "match": { "isLiteRTPieceList": true }, "value": false },
|
|
10
|
+
{ "match": {}, "value": true }
|
|
6
11
|
]
|
|
7
12
|
}
|
|
@@ -143,7 +143,7 @@ function maxBatchDecodeTokensSemantic(context) {
|
|
|
143
143
|
if (context.hasLinearAttentionLayers === true) {
|
|
144
144
|
return 32;
|
|
145
145
|
}
|
|
146
|
-
return context.hasGpuSplitPerLayerInputs === true ?
|
|
146
|
+
return context.hasGpuSplitPerLayerInputs === true ? 8 : null;
|
|
147
147
|
}
|
|
148
148
|
|
|
149
149
|
function enumerateMaxBatchDecodeTokenContexts() {
|
|
@@ -155,6 +155,24 @@ function enumerateMaxBatchDecodeTokenContexts() {
|
|
|
155
155
|
];
|
|
156
156
|
}
|
|
157
157
|
|
|
158
|
+
function prefillRecorderChunkLayersSemantic(context) {
|
|
159
|
+
if (context.hasGpuSplitPerLayerInputs === true && context.numTokens <= 32) {
|
|
160
|
+
return 8;
|
|
161
|
+
}
|
|
162
|
+
return 4;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function enumeratePrefillRecorderChunkLayerContexts() {
|
|
166
|
+
return [
|
|
167
|
+
{ hasGpuSplitPerLayerInputs: true, numTokens: 1 },
|
|
168
|
+
{ hasGpuSplitPerLayerInputs: true, numTokens: 15 },
|
|
169
|
+
{ hasGpuSplitPerLayerInputs: true, numTokens: 32 },
|
|
170
|
+
{ hasGpuSplitPerLayerInputs: true, numTokens: 33 },
|
|
171
|
+
{ hasGpuSplitPerLayerInputs: false, numTokens: 15 },
|
|
172
|
+
{ hasGpuSplitPerLayerInputs: false, numTokens: 64 },
|
|
173
|
+
];
|
|
174
|
+
}
|
|
175
|
+
|
|
158
176
|
function checkRuleShape(rules, expectedMatches, label) {
|
|
159
177
|
if (!Array.isArray(rules)) {
|
|
160
178
|
return {
|
|
@@ -223,6 +241,7 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
223
241
|
const profileDecodeRules = ruleGroup?.profileDecodeRecorderEnabled;
|
|
224
242
|
const batchRules = ruleGroup?.batchDecodeEnabled;
|
|
225
243
|
const maxBatchDecodeTokenRules = ruleGroup?.maxBatchDecodeTokens;
|
|
244
|
+
const prefillRecorderChunkLayerRules = ruleGroup?.prefillRecorderChunkLayers;
|
|
226
245
|
|
|
227
246
|
const decodeShape = checkRuleShape(
|
|
228
247
|
decodeRules,
|
|
@@ -358,9 +377,9 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
358
377
|
'[ExecutionRulesContract] maxBatchDecodeTokens linear-attention rule must cap unsafe burst recording at 32 tokens.'
|
|
359
378
|
);
|
|
360
379
|
}
|
|
361
|
-
if (!matchesExactObject(splitTablesRule?.match, { hasGpuSplitPerLayerInputs: true }) || splitTablesRule?.value !==
|
|
380
|
+
if (!matchesExactObject(splitTablesRule?.match, { hasGpuSplitPerLayerInputs: true }) || splitTablesRule?.value !== 8) {
|
|
362
381
|
maxBatchShapeErrors.push(
|
|
363
|
-
'[ExecutionRulesContract] maxBatchDecodeTokens enabling rule must cap gpu_split_tables bursts at
|
|
382
|
+
'[ExecutionRulesContract] maxBatchDecodeTokens enabling rule must cap gpu_split_tables bursts at 8 tokens.'
|
|
364
383
|
);
|
|
365
384
|
}
|
|
366
385
|
if (!matchesExactObject(fallbackRule?.match, {}) || fallbackRule?.value !== null) {
|
|
@@ -393,6 +412,54 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
393
412
|
ok: maxBatchSemantics.ok,
|
|
394
413
|
});
|
|
395
414
|
|
|
415
|
+
const prefillChunkShapeErrors = [];
|
|
416
|
+
if (!Array.isArray(prefillRecorderChunkLayerRules) || prefillRecorderChunkLayerRules.length !== 2) {
|
|
417
|
+
prefillChunkShapeErrors.push(
|
|
418
|
+
'[ExecutionRulesContract] prefillRecorderChunkLayers must contain exactly 2 rules.'
|
|
419
|
+
);
|
|
420
|
+
} else {
|
|
421
|
+
const [shortSplitTablesRule, fallbackRule] = prefillRecorderChunkLayerRules;
|
|
422
|
+
if (
|
|
423
|
+
!matchesExactObject(shortSplitTablesRule?.match, {
|
|
424
|
+
hasGpuSplitPerLayerInputs: true,
|
|
425
|
+
numTokens: { lte: 32 },
|
|
426
|
+
})
|
|
427
|
+
|| shortSplitTablesRule?.value !== 8
|
|
428
|
+
) {
|
|
429
|
+
prefillChunkShapeErrors.push(
|
|
430
|
+
'[ExecutionRulesContract] prefillRecorderChunkLayers short-prompt gpu_split_tables rule must use 8-layer chunks.'
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
if (!matchesExactObject(fallbackRule?.match, {}) || fallbackRule?.value !== 4) {
|
|
434
|
+
prefillChunkShapeErrors.push(
|
|
435
|
+
'[ExecutionRulesContract] prefillRecorderChunkLayers fallback rule must be { match: {}, value: 4 }.'
|
|
436
|
+
);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
errors.push(...prefillChunkShapeErrors);
|
|
440
|
+
checks.push({
|
|
441
|
+
id: 'inference.execution.prefillRecorderChunkLayers.shape',
|
|
442
|
+
ok: prefillChunkShapeErrors.length === 0,
|
|
443
|
+
});
|
|
444
|
+
|
|
445
|
+
const prefillChunkSemantics = Array.isArray(prefillRecorderChunkLayerRules)
|
|
446
|
+
? checkRuleSemantics(
|
|
447
|
+
prefillRecorderChunkLayerRules,
|
|
448
|
+
enumeratePrefillRecorderChunkLayerContexts(),
|
|
449
|
+
prefillRecorderChunkLayersSemantic,
|
|
450
|
+
'prefillRecorderChunkLayers'
|
|
451
|
+
)
|
|
452
|
+
: {
|
|
453
|
+
ok: false,
|
|
454
|
+
errors: ['[ExecutionRulesContract] prefillRecorderChunkLayers is unavailable for semantic check.'],
|
|
455
|
+
sampledContexts: 0,
|
|
456
|
+
};
|
|
457
|
+
errors.push(...prefillChunkSemantics.errors);
|
|
458
|
+
checks.push({
|
|
459
|
+
id: 'inference.execution.prefillRecorderChunkLayers.semantics',
|
|
460
|
+
ok: prefillChunkSemantics.ok,
|
|
461
|
+
});
|
|
462
|
+
|
|
396
463
|
return {
|
|
397
464
|
schemaVersion: 1,
|
|
398
465
|
source: 'doppler',
|
|
@@ -404,10 +471,12 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
404
471
|
profileDecodeRecorderRules: Array.isArray(profileDecodeRules) ? profileDecodeRules.length : 0,
|
|
405
472
|
batchDecodeRules: Array.isArray(batchRules) ? batchRules.length : 0,
|
|
406
473
|
maxBatchDecodeTokenRules: Array.isArray(maxBatchDecodeTokenRules) ? maxBatchDecodeTokenRules.length : 0,
|
|
474
|
+
prefillRecorderChunkLayerRules: Array.isArray(prefillRecorderChunkLayerRules) ? prefillRecorderChunkLayerRules.length : 0,
|
|
407
475
|
decodeRecorderContexts: decodeSemantics.sampledContexts,
|
|
408
476
|
profileDecodeRecorderContexts: profileDecodeSemantics.sampledContexts,
|
|
409
477
|
batchDecodeContexts: batchSemantics.sampledContexts,
|
|
410
478
|
maxBatchDecodeTokenContexts: maxBatchSemantics.sampledContexts,
|
|
479
|
+
prefillRecorderChunkLayerContexts: prefillChunkSemantics.sampledContexts,
|
|
411
480
|
},
|
|
412
481
|
};
|
|
413
482
|
}
|