doppler-gpu 0.4.7 → 0.4.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.md +93 -144
- package/models/catalog.json +1589 -0
- package/package.json +44 -12
- package/src/cli/doppler-cli.js +28 -7
- package/src/client/doppler-registry.json +2 -2
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +21 -5
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +6 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +9 -1
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/backward-registry.json +13 -2
- package/src/config/kernels/kernel-ref-digests.js +34 -21
- package/src/config/kernels/registry.json +1402 -2623
- package/src/config/merge.js +11 -0
- package/src/config/param-validator.js +25 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/doppler.schema.js +40 -5
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/schema/training-metrics.schema.js +4 -0
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +2 -2
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/attention-backward.js +123 -55
- package/src/experimental/training/autograd.js +101 -5
- package/src/experimental/training/clip.js +6 -1
- package/src/experimental/training/datasets/text-pairs.js +80 -8
- package/src/experimental/training/distillation/student-fixture.js +470 -114
- package/src/experimental/training/lora-pipeline.js +202 -27
- package/src/experimental/training/lora.js +10 -4
- package/src/experimental/training/optimizer.js +9 -1
- package/src/experimental/training/runner.js +18 -4
- package/src/experimental/training/suite.js +1 -1
- package/src/experimental/training/workloads.js +11 -0
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/backward/adam.js +6 -4
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
- package/src/gpu/kernels/backward/index.d.ts +3 -0
- package/src/gpu/kernels/backward/index.js +3 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
- package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
- package/src/gpu/kernels/backward/utils.d.ts +12 -2
- package/src/gpu/kernels/backward/utils.js +9 -2
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +26 -0
- package/src/gpu/kernels/index.js +18 -0
- package/src/gpu/kernels/linear-attention-core.js +413 -15
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +10 -1
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.d.ts +1 -0
- package/src/gpu/kernels/silu.js +19 -4
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +251 -20
- package/src/inference/browser-harness.js +221 -3
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +18 -0
- package/src/inference/pipelines/text/config.js +100 -0
- package/src/inference/pipelines/text/embed.js +1 -1
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +37 -0
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +130 -10
- package/src/inference/pipelines/text/ffn/dense.js +241 -21
- package/src/inference/pipelines/text/ffn/standard.js +121 -38
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +2 -0
- package/src/inference/pipelines/text/generator-runtime.js +1 -0
- package/src/inference/pipelines/text/generator-steps.js +2 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +326 -19
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +2 -2
- package/src/inference/pipelines/text/layer.js +87 -8
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +173 -16
- package/src/inference/pipelines/text/logits/gpu.js +2 -2
- package/src/inference/pipelines/text/logits/index.js +62 -1
- package/src/inference/pipelines/text/lora-apply.js +4 -4
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/moe-cpu.js +1 -0
- package/src/inference/pipelines/text/moe-gpu.js +1 -0
- package/src/inference/pipelines/text/ops.js +22 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +80 -11
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +40 -14
- package/src/inference/tokenizers/bundled.js +150 -55
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/inference/capability-transforms.rules.json +145 -20
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +2 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/node-converter.js +20 -5
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-experimental-exports.shared.d.ts +25 -0
- package/src/tooling-experimental-exports.shared.js +18 -0
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-runtime-map.svg +0 -82
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -54
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -62
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -319
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -33
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -31
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -306
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -166
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -152
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -69
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -26
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -93
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +0 -11
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -70
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -222
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -95,6 +95,67 @@ function resolveByteLevelPretokenizerConfig(preTokenizer) {
|
|
|
95
95
|
};
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
function normalizeInlineCaseInsensitiveGroups(pattern) {
|
|
99
|
+
let source = String(pattern || '');
|
|
100
|
+
let caseInsensitive = false;
|
|
101
|
+
let start = source.indexOf('(?i:');
|
|
102
|
+
while (start !== -1) {
|
|
103
|
+
let depth = 1;
|
|
104
|
+
let escaped = false;
|
|
105
|
+
let end = start + 4;
|
|
106
|
+
for (; end < source.length; end++) {
|
|
107
|
+
const char = source[end];
|
|
108
|
+
if (escaped) {
|
|
109
|
+
escaped = false;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (char === '\\') {
|
|
113
|
+
escaped = true;
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
if (char === '(') depth += 1;
|
|
117
|
+
else if (char === ')') {
|
|
118
|
+
depth -= 1;
|
|
119
|
+
if (depth === 0) break;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (depth !== 0) {
|
|
123
|
+
throw new Error('[Tokenizer] Unterminated inline case-insensitive regex group.');
|
|
124
|
+
}
|
|
125
|
+
source = `${source.slice(0, start)}(?:${source.slice(start + 4, end)})${source.slice(end + 1)}`;
|
|
126
|
+
caseInsensitive = true;
|
|
127
|
+
start = source.indexOf('(?i:', start + 3);
|
|
128
|
+
}
|
|
129
|
+
return { source, caseInsensitive };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function resolveIsolatedSplitPretokenizer(preTokenizer) {
|
|
133
|
+
if (!preTokenizer || typeof preTokenizer !== 'object') return null;
|
|
134
|
+
if (preTokenizer.type === 'Sequence' && Array.isArray(preTokenizer.pretokenizers)) {
|
|
135
|
+
for (const entry of preTokenizer.pretokenizers) {
|
|
136
|
+
const resolved = resolveIsolatedSplitPretokenizer(entry);
|
|
137
|
+
if (resolved) return resolved;
|
|
138
|
+
}
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
if (
|
|
142
|
+
preTokenizer.type !== 'Split'
|
|
143
|
+
|| preTokenizer.behavior !== 'Isolated'
|
|
144
|
+
|| preTokenizer.invert === true
|
|
145
|
+
|| typeof preTokenizer.pattern?.Regex !== 'string'
|
|
146
|
+
) {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
const normalized = normalizeInlineCaseInsensitiveGroups(preTokenizer.pattern.Regex);
|
|
150
|
+
const flags = `gu${normalized.caseInsensitive ? 'i' : ''}`;
|
|
151
|
+
try {
|
|
152
|
+
new RegExp(normalized.source, flags);
|
|
153
|
+
} catch (error) {
|
|
154
|
+
throw new Error(`[Tokenizer] Unsupported split pre-tokenizer regex: ${error.message}`);
|
|
155
|
+
}
|
|
156
|
+
return { source: normalized.source, flags };
|
|
157
|
+
}
|
|
158
|
+
|
|
98
159
|
function hexNibble(code) {
|
|
99
160
|
if (code >= 48 && code <= 57) return code - 48;
|
|
100
161
|
if (code >= 65 && code <= 70) return code - 55;
|
|
@@ -118,6 +179,28 @@ function parseByteTokenValue(token) {
|
|
|
118
179
|
return hi >= 0 && lo >= 0 ? (hi << 4) | lo : null;
|
|
119
180
|
}
|
|
120
181
|
|
|
182
|
+
function decodeByteFallbackTokens(tokens) {
|
|
183
|
+
const parts = [];
|
|
184
|
+
let bytes = [];
|
|
185
|
+
const flushBytes = () => {
|
|
186
|
+
if (bytes.length === 0) return;
|
|
187
|
+
parts.push(new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes)));
|
|
188
|
+
bytes = [];
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
for (const token of tokens) {
|
|
192
|
+
const byteValue = parseByteTokenValue(token);
|
|
193
|
+
if (byteValue !== null) {
|
|
194
|
+
bytes.push(byteValue);
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
flushBytes();
|
|
198
|
+
parts.push(token);
|
|
199
|
+
}
|
|
200
|
+
flushBytes();
|
|
201
|
+
return parts.join('');
|
|
202
|
+
}
|
|
203
|
+
|
|
121
204
|
function appendVocabEntry(token, id, vocab, reverseVocab, byteTokens) {
|
|
122
205
|
const numId = typeof id === 'number' ? id : parseInt( (id), 10);
|
|
123
206
|
vocab.set(token, numId);
|
|
@@ -366,6 +449,8 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
366
449
|
|
|
367
450
|
#useByteLevelEncoding = false;
|
|
368
451
|
|
|
452
|
+
#splitPretokenizer = null;
|
|
453
|
+
|
|
369
454
|
|
|
370
455
|
constructor(config = {}) {
|
|
371
456
|
// BundledTokenizer gets vocabSize from load(), so defer validation
|
|
@@ -390,6 +475,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
390
475
|
this.#byteDecoder = null;
|
|
391
476
|
this.#byteEncoder = null;
|
|
392
477
|
this.#useByteLevelEncoding = false;
|
|
478
|
+
this.#splitPretokenizer = null;
|
|
393
479
|
this.vocabSize = 0;
|
|
394
480
|
}
|
|
395
481
|
|
|
@@ -469,9 +555,6 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
469
555
|
}
|
|
470
556
|
log.info('Tokenizer', `HuggingFace model.type="${model.type}", using type="${this.#type}"`);
|
|
471
557
|
this.#byteDecoder = null;
|
|
472
|
-
if (this.#type === 'bpe') {
|
|
473
|
-
this.#initializeByteDecoder();
|
|
474
|
-
}
|
|
475
558
|
let maxId = -1;
|
|
476
559
|
|
|
477
560
|
// Handle vocab based on type
|
|
@@ -555,6 +638,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
555
638
|
// Handle behavior flags (use HF config if present, else runtime defaults)
|
|
556
639
|
const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
|
|
557
640
|
const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(hf.pre_tokenizer);
|
|
641
|
+
this.#splitPretokenizer = resolveIsolatedSplitPretokenizer(hf.pre_tokenizer);
|
|
558
642
|
const configuredAddBosToken = this.addBosToken;
|
|
559
643
|
const configuredAddEosToken = this.addEosToken;
|
|
560
644
|
const inferredFlags = inferBundledTokenizerBehaviorFlags(hf, this.specialTokens);
|
|
@@ -586,6 +670,9 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
586
670
|
const decoderPrepend = hf.decoder?.prepend_scheme === 'always' || hf.decoder?.add_prefix_space === true;
|
|
587
671
|
const normalizerPrepend = hf.normalizer?.prepend_scheme === 'always' || hf.normalizer?.add_prefix_space === true;
|
|
588
672
|
this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
|
|
673
|
+
if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
|
|
674
|
+
this.#initializeByteDecoder();
|
|
675
|
+
}
|
|
589
676
|
const runtimeSpacePrefix = runtimeDefaults.addSpacePrefix;
|
|
590
677
|
// Use explicit runtime config if set (non-null), otherwise auto-detect from tokenizer.json
|
|
591
678
|
this.#addSpacePrefix = runtimeSpacePrefix
|
|
@@ -646,9 +733,6 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
646
733
|
throw new Error(`[Tokenizer] Unsupported tokenizer type: ${tokenizerJson.type}`);
|
|
647
734
|
}
|
|
648
735
|
this.#byteDecoder = null;
|
|
649
|
-
if (this.#type === 'bpe') {
|
|
650
|
-
this.#initializeByteDecoder();
|
|
651
|
-
}
|
|
652
736
|
|
|
653
737
|
// Build vocab maps
|
|
654
738
|
let maxId = loadObjectVocab(tokenizerJson.vocab, this.#vocab, this.#reverseVocab, this.#byteTokens);
|
|
@@ -722,6 +806,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
722
806
|
|
|
723
807
|
const runtimeDefaults = getRuntimeConfig().inference.tokenizer;
|
|
724
808
|
const byteLevelPretokenizer = resolveByteLevelPretokenizerConfig(tokenizerJson.pre_tokenizer);
|
|
809
|
+
this.#splitPretokenizer = resolveIsolatedSplitPretokenizer(tokenizerJson.pre_tokenizer);
|
|
725
810
|
const configuredAddBosToken = this.addBosToken;
|
|
726
811
|
const configuredAddEosToken = this.addEosToken;
|
|
727
812
|
const inferredFlags = inferBundledTokenizerBehaviorFlags(tokenizerJson, this.specialTokens);
|
|
@@ -744,6 +829,9 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
744
829
|
throw new Error('[Tokenizer] addEosToken is enabled but eos token is missing.');
|
|
745
830
|
}
|
|
746
831
|
this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
|
|
832
|
+
if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
|
|
833
|
+
this.#initializeByteDecoder();
|
|
834
|
+
}
|
|
747
835
|
// NOTE: Default to FALSE - first word shouldn't get space prefix
|
|
748
836
|
// Space prefixes are only for words that follow a space in original text
|
|
749
837
|
this.#addSpacePrefix = tokenizerJson.addSpacePrefix === true
|
|
@@ -931,48 +1019,55 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
931
1019
|
if (text.length === 0) return [];
|
|
932
1020
|
|
|
933
1021
|
let normalized = text;
|
|
934
|
-
let prefixed;
|
|
935
|
-
if (this.#useByteLevelEncoding) {
|
|
936
|
-
if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
|
|
937
|
-
normalized = ` ${normalized}`;
|
|
938
|
-
}
|
|
939
|
-
prefixed = this.#encodeByteLevelText(normalized);
|
|
940
|
-
} else {
|
|
941
|
-
if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
|
|
942
|
-
normalized = ` ${normalized}`;
|
|
943
|
-
}
|
|
944
|
-
const sp = this.#spacePrefixChar;
|
|
945
|
-
prefixed = normalized.replace(/ /g, sp);
|
|
946
|
-
}
|
|
947
|
-
|
|
948
|
-
if (this.#mergeRanks.size === 0) {
|
|
949
|
-
return this.#encodeBPEGreedy(prefixed);
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
const tokens = this.#bpeTokenize(prefixed);
|
|
953
|
-
|
|
954
1022
|
const ids = [];
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
1023
|
+
if (this.#addSpacePrefix && !normalized.startsWith(' ')) {
|
|
1024
|
+
normalized = ` ${normalized}`;
|
|
1025
|
+
}
|
|
1026
|
+
const pretokens = this.#splitBpePretokens(normalized);
|
|
1027
|
+
for (const pretoken of pretokens) {
|
|
1028
|
+
const encoded = this.#useByteLevelEncoding
|
|
1029
|
+
? this.#encodeByteLevelText(pretoken)
|
|
1030
|
+
: pretoken.replace(/ /g, this.#spacePrefixChar);
|
|
1031
|
+
if (this.#mergeRanks.size === 0) {
|
|
1032
|
+
ids.push(...this.#encodeBPEGreedy(encoded));
|
|
959
1033
|
continue;
|
|
960
1034
|
}
|
|
961
|
-
const
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
ids.push(byteId);
|
|
1035
|
+
for (const token of this.#bpeTokenize(encoded)) {
|
|
1036
|
+
const id = this.#vocab.get(token);
|
|
1037
|
+
if (id !== undefined) {
|
|
1038
|
+
ids.push(id);
|
|
966
1039
|
continue;
|
|
967
1040
|
}
|
|
968
|
-
const
|
|
969
|
-
|
|
1041
|
+
const bytes = new TextEncoder().encode(token);
|
|
1042
|
+
for (const b of bytes) {
|
|
1043
|
+
const byteId = this.#byteTokens.get(b);
|
|
1044
|
+
if (byteId !== undefined) {
|
|
1045
|
+
ids.push(byteId);
|
|
1046
|
+
continue;
|
|
1047
|
+
}
|
|
1048
|
+
const byteToken = `<0x${b.toString(16).padStart(2, '0').toUpperCase()}>`;
|
|
1049
|
+
ids.push(this.#vocab.get(byteToken) ?? this.#getUnkTokenId());
|
|
1050
|
+
}
|
|
970
1051
|
}
|
|
971
1052
|
}
|
|
972
|
-
|
|
973
1053
|
return ids;
|
|
974
1054
|
}
|
|
975
1055
|
|
|
1056
|
+
#splitBpePretokens(text) {
|
|
1057
|
+
if (!this.#useByteLevelEncoding || !this.#splitPretokenizer) return [text];
|
|
1058
|
+
const regex = new RegExp(this.#splitPretokenizer.source, this.#splitPretokenizer.flags);
|
|
1059
|
+
const pretokens = [];
|
|
1060
|
+
let cursor = 0;
|
|
1061
|
+
for (const match of text.matchAll(regex)) {
|
|
1062
|
+
const index = match.index ?? cursor;
|
|
1063
|
+
if (index > cursor) pretokens.push(text.slice(cursor, index));
|
|
1064
|
+
if (match[0]) pretokens.push(match[0]);
|
|
1065
|
+
cursor = index + match[0].length;
|
|
1066
|
+
}
|
|
1067
|
+
if (cursor < text.length) pretokens.push(text.slice(cursor));
|
|
1068
|
+
return pretokens.length > 0 ? pretokens : [text];
|
|
1069
|
+
}
|
|
1070
|
+
|
|
976
1071
|
|
|
977
1072
|
#encodeBPEGreedy(text) {
|
|
978
1073
|
|
|
@@ -1073,29 +1168,29 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
1073
1168
|
|
|
1074
1169
|
const token = this.#reverseVocab.get(id);
|
|
1075
1170
|
if (token !== undefined) {
|
|
1076
|
-
|
|
1077
|
-
if (token.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
|
|
1078
|
-
const byteVal = parseInt(token.slice(3, 5), 16);
|
|
1079
|
-
tokens.push(String.fromCharCode(byteVal));
|
|
1080
|
-
} else {
|
|
1081
|
-
tokens.push(token);
|
|
1082
|
-
}
|
|
1171
|
+
tokens.push(token);
|
|
1083
1172
|
}
|
|
1084
1173
|
}
|
|
1085
1174
|
|
|
1086
1175
|
let result;
|
|
1087
|
-
if (this.#type === 'bpe' && this.#
|
|
1088
|
-
const merged = tokens.join('');
|
|
1176
|
+
if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
|
|
1089
1177
|
const bytes = [];
|
|
1090
|
-
for (const
|
|
1091
|
-
const
|
|
1092
|
-
if (
|
|
1093
|
-
bytes.push(
|
|
1178
|
+
for (const token of tokens) {
|
|
1179
|
+
const byteValue = parseByteTokenValue(token);
|
|
1180
|
+
if (byteValue !== null) {
|
|
1181
|
+
bytes.push(byteValue);
|
|
1094
1182
|
continue;
|
|
1095
1183
|
}
|
|
1096
|
-
const
|
|
1097
|
-
|
|
1098
|
-
|
|
1184
|
+
for (const ch of token) {
|
|
1185
|
+
const mapped = this.#byteDecoder.get(ch);
|
|
1186
|
+
if (mapped != null) {
|
|
1187
|
+
bytes.push(mapped);
|
|
1188
|
+
continue;
|
|
1189
|
+
}
|
|
1190
|
+
const fallbackBytes = new TextEncoder().encode(ch);
|
|
1191
|
+
for (const byte of fallbackBytes) {
|
|
1192
|
+
bytes.push(byte);
|
|
1193
|
+
}
|
|
1099
1194
|
}
|
|
1100
1195
|
}
|
|
1101
1196
|
result = new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes));
|
|
@@ -1103,7 +1198,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
1103
1198
|
result = result.replace(/▁/g, ' ');
|
|
1104
1199
|
} else {
|
|
1105
1200
|
// Join and convert ▁ back to spaces, handle GPT-style markers
|
|
1106
|
-
result = tokens
|
|
1201
|
+
result = decodeByteFallbackTokens(tokens)
|
|
1107
1202
|
.replace(/▁/g, ' ')
|
|
1108
1203
|
.replace(/Ġ/g, ' ')
|
|
1109
1204
|
.replace(/Ċ/g, '\n');
|
|
@@ -4,9 +4,13 @@
|
|
|
4
4
|
* @module inference/tokenizers/types
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
export interface SpecialTokens {
|
|
8
|
+
bos?: number;
|
|
9
|
+
eos?: number;
|
|
10
|
+
pad?: number;
|
|
11
|
+
unk?: number;
|
|
12
|
+
mask?: number;
|
|
13
|
+
}
|
|
10
14
|
|
|
11
15
|
/** Tokenizer Configuration */
|
|
12
16
|
export interface TokenizerConfig {
|
|
@@ -144,4 +148,3 @@ export interface BundledTokenizerJson {
|
|
|
144
148
|
addEosToken?: boolean;
|
|
145
149
|
addSpacePrefix?: boolean;
|
|
146
150
|
}
|
|
147
|
-
|
|
@@ -35,6 +35,7 @@ import { buildTensorLocations } from './shard-resolver.js';
|
|
|
35
35
|
import {
|
|
36
36
|
needsNormWeightOffset,
|
|
37
37
|
resolveWeightLayout,
|
|
38
|
+
requiresCpuF16ToF32MatmulMaterialization,
|
|
38
39
|
shouldStreamLargeWeight,
|
|
39
40
|
} from './manifest-config.js';
|
|
40
41
|
import { MemoryMonitor } from './memory-monitor.js';
|
|
@@ -119,7 +120,7 @@ function createLoadTiming(modelId, hasCustomLoader) {
|
|
|
119
120
|
status: 'running',
|
|
120
121
|
customShardLoader: hasCustomLoader === true,
|
|
121
122
|
byteAccountingMode: hasCustomLoader === true
|
|
122
|
-
? 'custom-loader-progress
|
|
123
|
+
? 'custom-loader-read-progress'
|
|
123
124
|
: 'full-shard-progress',
|
|
124
125
|
totalBytes: null,
|
|
125
126
|
totalShards: null,
|
|
@@ -588,10 +589,21 @@ export class DopplerLoader {
|
|
|
588
589
|
const loadStartTime = Date.now();
|
|
589
590
|
let bytesLoaded = 0;
|
|
590
591
|
let shardsLoaded = 0;
|
|
592
|
+
this.shardCache.resetCustomReadStats();
|
|
593
|
+
|
|
594
|
+
const syncCustomReadStats = () => {
|
|
595
|
+
if (!this.shardCache.hasCustomLoader) return;
|
|
596
|
+
const stats = this.shardCache.customReadStats;
|
|
597
|
+
bytesLoaded = stats.bytesRead;
|
|
598
|
+
shardsLoaded = stats.shardsRead;
|
|
599
|
+
this.loadTiming.bytesLoaded = bytesLoaded;
|
|
600
|
+
this.loadTiming.shardsLoaded = shardsLoaded;
|
|
601
|
+
};
|
|
591
602
|
|
|
592
603
|
|
|
593
604
|
const reportProgress = (stage, baseProgress, detail) => {
|
|
594
605
|
if (!onProgress || typeof onProgress !== 'function') return;
|
|
606
|
+
syncCustomReadStats();
|
|
595
607
|
const elapsed = (Date.now() - loadStartTime) / 1000;
|
|
596
608
|
const speed = elapsed > 0 ? bytesLoaded / elapsed : 0;
|
|
597
609
|
const speedStr = speed > 0 ? `${formatBytes(speed)}/s` : '';
|
|
@@ -750,6 +762,7 @@ export class DopplerLoader {
|
|
|
750
762
|
}
|
|
751
763
|
|
|
752
764
|
if (onProgress) {
|
|
765
|
+
syncCustomReadStats();
|
|
753
766
|
const layerFraction = (l + 1) / numLayers;
|
|
754
767
|
const layerProgress = 0.80 + layerFraction * 0.05;
|
|
755
768
|
onProgress({
|
|
@@ -785,9 +798,17 @@ export class DopplerLoader {
|
|
|
785
798
|
await this.#loadFinalWeights(onProgress);
|
|
786
799
|
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
787
800
|
this.#assertResidentBudget('final weights');
|
|
801
|
+
syncCustomReadStats();
|
|
788
802
|
|
|
789
803
|
if (onProgress) {
|
|
790
|
-
onProgress({
|
|
804
|
+
onProgress({
|
|
805
|
+
stage: 'complete',
|
|
806
|
+
progress: 1.0,
|
|
807
|
+
shard: shardsLoaded,
|
|
808
|
+
totalShards,
|
|
809
|
+
bytesLoaded,
|
|
810
|
+
totalBytes,
|
|
811
|
+
});
|
|
791
812
|
}
|
|
792
813
|
|
|
793
814
|
this.isLoaded = true;
|
|
@@ -804,6 +825,7 @@ export class DopplerLoader {
|
|
|
804
825
|
return (this.manifest.config) || {};
|
|
805
826
|
} catch (error) {
|
|
806
827
|
loadError = error;
|
|
828
|
+
syncCustomReadStats();
|
|
807
829
|
finishLoadTiming(this.loadTiming, 'failed', loadTimingStart, error, activeLoadPhase);
|
|
808
830
|
} finally {
|
|
809
831
|
this.#loadShardOverride = null;
|
|
@@ -1157,6 +1179,7 @@ export class DopplerLoader {
|
|
|
1157
1179
|
if (!location?.size || location.size <= 0) return false;
|
|
1158
1180
|
if (hasSourceTransform(location)) return false;
|
|
1159
1181
|
if (Array.isArray(location?.storage?.companions) && location.storage.companions.length > 0) return false;
|
|
1182
|
+
if (requiresCpuF16ToF32MatmulMaterialization(location, this.gpuCapabilities, this.keepF32Weights)) return false;
|
|
1160
1183
|
if (this.shardCache.hasCustomLoader && !this.shardCache.canStreamRanges) return false;
|
|
1161
1184
|
const chunkBytes = this.#loadingConfig?.storage?.backend?.streaming?.readChunkBytes ?? 0;
|
|
1162
1185
|
if (!Number.isFinite(chunkBytes) || chunkBytes <= 0) return false;
|
package/src/loader/index.d.ts
CHANGED
|
@@ -42,11 +42,13 @@ export {
|
|
|
42
42
|
getLargeWeightConfig,
|
|
43
43
|
getLargeWeightMaxBytes,
|
|
44
44
|
estimateMatmulWeightBytes,
|
|
45
|
+
resolveMatmulWeightDtype,
|
|
46
|
+
requiresCpuF16ToF32MatmulMaterialization,
|
|
45
47
|
resolveWeightLayout,
|
|
46
48
|
shouldStreamLargeWeight,
|
|
47
49
|
isMoEModel,
|
|
48
50
|
} from './manifest-config.js';
|
|
49
|
-
export type { LargeWeightConfig } from '
|
|
51
|
+
export type { LargeWeightConfigSchema as LargeWeightConfig } from '../config/schema/doppler.schema.js';
|
|
50
52
|
|
|
51
53
|
export { buildTensorLocations } from './shard-resolver.js';
|
|
52
54
|
export type { BuildTensorLocationsOptions } from './shard-resolver.js';
|
package/src/loader/index.js
CHANGED
|
@@ -45,6 +45,8 @@ export interface LayerWeights {
|
|
|
45
45
|
qkvProj?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
46
46
|
qkvSizes?: [number, number, number] | null;
|
|
47
47
|
qkvDtype?: 'f16' | 'f32' | null;
|
|
48
|
+
linearABProj?: WeightBuffer | null;
|
|
49
|
+
linearQKVZProj?: WeightBuffer | null;
|
|
48
50
|
linearInProjZ?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
49
51
|
linearInProjA?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
50
52
|
linearInProjB?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
@@ -158,7 +160,7 @@ export interface LoaderLoadTiming {
|
|
|
158
160
|
modelId: string | null;
|
|
159
161
|
status: 'running' | 'complete' | 'failed';
|
|
160
162
|
customShardLoader: boolean;
|
|
161
|
-
byteAccountingMode: 'full-shard-progress' | 'custom-loader-progress
|
|
163
|
+
byteAccountingMode: 'full-shard-progress' | 'custom-loader-read-progress';
|
|
162
164
|
totalBytes: number | null;
|
|
163
165
|
totalShards: number | null;
|
|
164
166
|
bytesLoaded: number;
|
|
@@ -59,6 +59,24 @@ export declare function estimateMatmulWeightBytes(
|
|
|
59
59
|
keepF32Weights: boolean
|
|
60
60
|
): { bytes: number; dtype: WeightDtype } | null;
|
|
61
61
|
|
|
62
|
+
/**
|
|
63
|
+
* Resolve the materialized dtype for a weight-like tensor.
|
|
64
|
+
*/
|
|
65
|
+
export declare function resolveMatmulWeightDtype(
|
|
66
|
+
location: TensorLocation,
|
|
67
|
+
gpuCapabilities: KernelCapabilities | null,
|
|
68
|
+
keepF32Weights: boolean
|
|
69
|
+
): WeightDtype;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Check whether an F16 tensor must be widened on CPU before GPU upload.
|
|
73
|
+
*/
|
|
74
|
+
export declare function requiresCpuF16ToF32MatmulMaterialization(
|
|
75
|
+
location: TensorLocation,
|
|
76
|
+
gpuCapabilities: KernelCapabilities | null,
|
|
77
|
+
keepF32Weights: boolean
|
|
78
|
+
): boolean;
|
|
79
|
+
|
|
62
80
|
/**
|
|
63
81
|
* Resolve weight layout from tensor location.
|
|
64
82
|
*
|
|
@@ -67,19 +67,27 @@ export function estimateMatmulWeightBytes(location, gpuCapabilities, keepF32Weig
|
|
|
67
67
|
const numElements = location.shape.reduce((a, b) => a * b, 1);
|
|
68
68
|
if (!Number.isFinite(numElements) || numElements <= 0) return null;
|
|
69
69
|
|
|
70
|
+
const dtype = resolveMatmulWeightDtype(location, gpuCapabilities, keepF32Weights);
|
|
71
|
+
const bytesPerElement = DTYPE_SIZES[selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype })];
|
|
72
|
+
return { bytes: numElements * bytesPerElement, dtype };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function resolveMatmulWeightDtype(location, gpuCapabilities, keepF32Weights) {
|
|
70
76
|
const caps = gpuCapabilities || getKernelCapabilities();
|
|
71
77
|
const hasF16 = caps?.hasF16 ?? false;
|
|
72
78
|
const isMatmulWeight = shouldDequantizeToF16(location);
|
|
73
79
|
|
|
74
|
-
|
|
80
|
+
return selectRuleValue('loader', 'weights', 'matmulWeightDtype', {
|
|
75
81
|
locationDtype: location.dtype,
|
|
76
82
|
hasF16,
|
|
77
83
|
isMatmulWeight,
|
|
78
84
|
keepF32Weights: Boolean(keepF32Weights),
|
|
79
85
|
});
|
|
86
|
+
}
|
|
80
87
|
|
|
81
|
-
|
|
82
|
-
|
|
88
|
+
export function requiresCpuF16ToF32MatmulMaterialization(location, gpuCapabilities, keepF32Weights) {
|
|
89
|
+
if (location?.dtype !== 'F16' || !location?.role) return false;
|
|
90
|
+
return resolveMatmulWeightDtype(location, gpuCapabilities, keepF32Weights) === 'f32';
|
|
83
91
|
}
|
|
84
92
|
|
|
85
93
|
export function resolveWeightLayout(location) {
|
|
@@ -39,6 +39,8 @@ export class ShardCache {
|
|
|
39
39
|
get hasCustomRangeLoader(): boolean;
|
|
40
40
|
get hasCustomStreamLoader(): boolean;
|
|
41
41
|
get canStreamRanges(): boolean;
|
|
42
|
+
resetCustomReadStats(): void;
|
|
43
|
+
get customReadStats(): { bytesRead: number; shardsRead: number };
|
|
42
44
|
has(shardIndex: number): boolean;
|
|
43
45
|
get size(): number;
|
|
44
46
|
get totalBytes(): number;
|
|
@@ -25,6 +25,8 @@ export class ShardCache {
|
|
|
25
25
|
#highPriorityQueue = [];
|
|
26
26
|
#lowPriorityQueue = [];
|
|
27
27
|
#epoch = 0;
|
|
28
|
+
#customReadBytes = 0;
|
|
29
|
+
#customReadShards = new Set();
|
|
28
30
|
|
|
29
31
|
lastSource = null;
|
|
30
32
|
|
|
@@ -114,6 +116,18 @@ export class ShardCache {
|
|
|
114
116
|
}
|
|
115
117
|
}
|
|
116
118
|
|
|
119
|
+
resetCustomReadStats() {
|
|
120
|
+
this.#customReadBytes = 0;
|
|
121
|
+
this.#customReadShards.clear();
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
get customReadStats() {
|
|
125
|
+
return {
|
|
126
|
+
bytesRead: this.#customReadBytes,
|
|
127
|
+
shardsRead: this.#customReadShards.size,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
117
131
|
setManifest(manifest) {
|
|
118
132
|
this.#manifest = manifest;
|
|
119
133
|
}
|
|
@@ -228,6 +242,13 @@ export class ShardCache {
|
|
|
228
242
|
throw new Error('Custom shard loader must return ArrayBuffer or Uint8Array.');
|
|
229
243
|
}
|
|
230
244
|
|
|
245
|
+
#recordCustomRead(shardIndex, byteLength) {
|
|
246
|
+
const bytes = Number(byteLength);
|
|
247
|
+
if (!Number.isFinite(bytes) || bytes <= 0) return;
|
|
248
|
+
this.#customReadBytes += Math.floor(bytes);
|
|
249
|
+
this.#customReadShards.add(shardIndex);
|
|
250
|
+
}
|
|
251
|
+
|
|
231
252
|
#throwShortStreamRead(shardIndex, start, want, produced, path) {
|
|
232
253
|
throw new Error(
|
|
233
254
|
`Shard ${shardIndex} short stream read via ${path}: ` +
|
|
@@ -257,7 +278,9 @@ export class ShardCache {
|
|
|
257
278
|
const rangeData = await this.#customRangeLoader(shardIndex, start, want);
|
|
258
279
|
const elapsed = (performance.now() - rangeStart) / 1000;
|
|
259
280
|
this.#setLastSource('custom', elapsed, 'range', 'custom-range');
|
|
260
|
-
|
|
281
|
+
const buffer = this.#toArrayBuffer(rangeData);
|
|
282
|
+
this.#recordCustomRead(shardIndex, buffer.byteLength);
|
|
283
|
+
return buffer;
|
|
261
284
|
} catch (error) {
|
|
262
285
|
const unsupported = this.#isUnsupportedRangeOrStream(error);
|
|
263
286
|
if (!unsupported) {
|
|
@@ -337,6 +360,7 @@ export class ShardCache {
|
|
|
337
360
|
const bytes = chunk instanceof Uint8Array ? chunk : new Uint8Array(this.#toArrayBuffer(chunk));
|
|
338
361
|
if (bytes.byteLength > 0) {
|
|
339
362
|
produced += bytes.byteLength;
|
|
363
|
+
this.#recordCustomRead(shardIndex, bytes.byteLength);
|
|
340
364
|
yield bytes;
|
|
341
365
|
}
|
|
342
366
|
}
|
|
@@ -367,6 +391,7 @@ export class ShardCache {
|
|
|
367
391
|
}
|
|
368
392
|
emptyRetries = 0;
|
|
369
393
|
resumed += bytes.byteLength;
|
|
394
|
+
this.#recordCustomRead(shardIndex, bytes.byteLength);
|
|
370
395
|
yield bytes;
|
|
371
396
|
}
|
|
372
397
|
if (want != null && produced + resumed < want) {
|
|
@@ -411,6 +436,7 @@ export class ShardCache {
|
|
|
411
436
|
}
|
|
412
437
|
emptyRetries = 0;
|
|
413
438
|
resumed += bytes.byteLength;
|
|
439
|
+
this.#recordCustomRead(shardIndex, bytes.byteLength);
|
|
414
440
|
yield bytes;
|
|
415
441
|
}
|
|
416
442
|
if (produced + resumed < want) {
|
|
@@ -462,6 +488,7 @@ export class ShardCache {
|
|
|
462
488
|
}
|
|
463
489
|
emptyRetries = 0;
|
|
464
490
|
produced += bytes.byteLength;
|
|
491
|
+
this.#recordCustomRead(shardIndex, bytes.byteLength);
|
|
465
492
|
yield bytes;
|
|
466
493
|
if (bytes.byteLength < requestLength) {
|
|
467
494
|
partialRetryUsed = true;
|
|
@@ -528,6 +555,7 @@ export class ShardCache {
|
|
|
528
555
|
|
|
529
556
|
// Normalize to ArrayBuffer for downstream slicing
|
|
530
557
|
const arrayBuffer = this.#toArrayBuffer(data);
|
|
558
|
+
this.#recordCustomRead(shardIndex, arrayBuffer.byteLength);
|
|
531
559
|
|
|
532
560
|
if (epoch === this.#epoch) {
|
|
533
561
|
this.#add(shardIndex, arrayBuffer);
|
package/src/models/gemma4.js
CHANGED
|
@@ -23,6 +23,14 @@ const family = createFamily({
|
|
|
23
23
|
defaultRuntimeProfile: 'profiles/throughput',
|
|
24
24
|
modes: ['text', 'vision'],
|
|
25
25
|
},
|
|
26
|
+
{
|
|
27
|
+
modelId: 'gemma-4-e2b-it-q4k-ehf16-af16-int4ple',
|
|
28
|
+
label: 'Gemma 4 E2B Instruct (Q4K/F16a/INT4 PLE)',
|
|
29
|
+
sourceModel: 'google/gemma-4-e2b-it',
|
|
30
|
+
hfPath: 'models/gemma-4-e2b-it-q4k-ehf16-af16-int4ple',
|
|
31
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
32
|
+
modes: ['text', 'vision'],
|
|
33
|
+
},
|
|
26
34
|
{
|
|
27
35
|
modelId: 'gemma-4-12b-it-text-w4a16-ct-ehf16-af16',
|
|
28
36
|
label: 'Gemma 4 12B Instruct Text (W4A16 QAT/F16a)',
|