doppler-gpu 0.4.6 → 0.4.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +98 -137
- package/assets/doppler.svg +1 -3
- package/models/catalog.json +1575 -0
- package/package.json +59 -26
- package/src/cli/doppler-cli.js +28 -7
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +52 -3
- package/src/client/runtime/index.js +25 -2
- package/src/client/runtime/model-session.d.ts +31 -1
- package/src/client/runtime/model-session.js +31 -0
- package/src/client/runtime/model-source.d.ts +19 -2
- package/src/client/runtime/model-source.js +45 -5
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +50 -9
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +35 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +548 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/kernel-path-loader.js +1 -0
- package/src/config/kernels/kernel-ref-digests.js +26 -17
- package/src/config/kernels/registry.json +448 -27
- package/src/config/merge.js +41 -0
- package/src/config/param-validator.js +67 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/runtime/default.json +3 -1
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +20 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +18 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +52 -0
- package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +85 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +23 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +38 -0
- package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +135 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +37 -0
- package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +21 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +41 -0
- package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +41 -0
- package/src/config/runtime/profiles/rerank-stability.json +43 -0
- package/src/config/runtime/profiles/rerank-throughput.json +40 -0
- package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +22 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +117 -292
- package/src/config/schema/execution-v1.schema.d.ts +41 -0
- package/src/config/schema/execution-v1.schema.js +10 -0
- package/src/config/schema/index.d.ts +3 -1
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/speculation-self.schema.d.ts +16 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/capability-transform-resolver.js +7 -1
- package/src/config/transforms/execution-graph-transforms.js +10 -2
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/debug/config.d.ts +4 -0
- package/src/experimental/orchestration/index.d.ts +10 -1
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +2 -2
- package/src/formats/rdrr/classification.js +3 -0
- package/src/formats/rdrr/integrity.d.ts +0 -2
- package/src/formats/rdrr/integrity.js +0 -2
- package/src/formats/rdrr/types.d.ts +1 -1
- package/src/gpu/kernels/attention.d.ts +2 -0
- package/src/gpu/kernels/attention.js +75 -16
- package/src/gpu/kernels/attention_decode_online_head256_f16kv_output_gate.wgsl +268 -0
- package/src/gpu/kernels/attention_head128_f16kv.wgsl +269 -0
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/fused_ffn.d.ts +32 -0
- package/src/gpu/kernels/fused_ffn.js +540 -19
- package/src/gpu/kernels/fused_ffn.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_f16w_f32a_batched.wgsl +135 -0
- package/src/gpu/kernels/fused_ffn_q4k.wgsl +5 -3
- package/src/gpu/kernels/fused_ffn_q4k_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_ffn_q4k_metal_simd16.wgsl +183 -0
- package/src/gpu/kernels/fused_gate_up_gelu_gated_f16.wgsl +2 -1
- package/src/gpu/kernels/fused_matmul_q4.wgsl +15 -6
- package/src/gpu/kernels/fused_matmul_q4_metal_simd16.wgsl +144 -0
- package/src/gpu/kernels/index.d.ts +21 -0
- package/src/gpu/kernels/index.js +13 -0
- package/src/gpu/kernels/linear-attention-core.js +414 -21
- package/src/gpu/kernels/lm-head-argmax.d.ts +10 -1
- package/src/gpu/kernels/lm-head-argmax.js +153 -57
- package/src/gpu/kernels/lm-head-select-logits.d.ts +23 -0
- package/src/gpu/kernels/lm-head-select-logits.js +186 -0
- package/src/gpu/kernels/lm_head_argmax_q4k.wgsl +289 -0
- package/src/gpu/kernels/lm_head_select_logits.wgsl +77 -0
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/residual_rmsnorm_pair.wgsl +127 -0
- package/src/gpu/kernels/rmsnorm-pair.d.ts +33 -0
- package/src/gpu/kernels/rmsnorm-pair.js +113 -0
- package/src/gpu/kernels/rmsnorm-stats.d.ts +29 -0
- package/src/gpu/kernels/rmsnorm-stats.js +250 -0
- package/src/gpu/kernels/silu.wgsl +7 -1
- package/src/gpu/kernels/silu_f16.wgsl +7 -1
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +15 -10
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +37 -7
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +37 -7
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/gpu/tensor.d.ts +1 -0
- package/src/index.d.ts +1 -1
- package/src/index.js +1 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-suite-helpers.js +1 -1
- package/src/inference/browser-harness-text-helpers.js +589 -117
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +486 -13
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/base.js +42 -1
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +18 -0
- package/src/inference/pipelines/text/attention/dispatch-params.js +8 -3
- package/src/inference/pipelines/text/attention/output-gate-fusion.js +37 -0
- package/src/inference/pipelines/text/attention/projections.js +174 -4
- package/src/inference/pipelines/text/attention/record.js +44 -21
- package/src/inference/pipelines/text/attention/run.js +18 -3
- package/src/inference/pipelines/text/config.d.ts +23 -1
- package/src/inference/pipelines/text/config.js +235 -112
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.d.ts +2 -0
- package/src/inference/pipelines/text/execution-plan.js +45 -2
- package/src/inference/pipelines/text/execution-v1.d.ts +6 -0
- package/src/inference/pipelines/text/execution-v1.js +122 -8
- package/src/inference/pipelines/text/ffn/dense.js +254 -25
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.js +145 -39
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -0
- package/src/inference/pipelines/text/generator-helpers.js +4 -1
- package/src/inference/pipelines/text/generator-runtime.js +54 -20
- package/src/inference/pipelines/text/generator-steps.js +5 -2
- package/src/inference/pipelines/text/generator.d.ts +18 -1
- package/src/inference/pipelines/text/generator.js +303 -20
- package/src/inference/pipelines/text/init-qkv-fusion.js +155 -62
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.d.ts +22 -3
- package/src/inference/pipelines/text/init.js +65 -17
- package/src/inference/pipelines/text/layer.js +111 -5
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +208 -0
- package/src/inference/pipelines/text/linear-attention.d.ts +1 -0
- package/src/inference/pipelines/text/linear-attention.js +185 -24
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +78 -115
- package/src/inference/pipelines/text/logits/index.js +105 -87
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/model-load.js +6 -0
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.js +48 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +105 -21
- package/src/inference/pipelines/text/probes.js +9 -4
- package/src/inference/pipelines/text/state.d.ts +4 -4
- package/src/inference/pipelines/text/state.js +4 -0
- package/src/inference/pipelines/text/types.d.ts +27 -2
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.d.ts +21 -1
- package/src/inference/pipelines/text.js +60 -14
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/inference/tokenizers/bundled.js +45 -23
- package/src/inference/tokenizers/types.d.ts +7 -4
- package/src/loader/doppler-loader.js +25 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/index.d.ts +3 -1
- package/src/loader/index.js +2 -0
- package/src/loader/loader-types.d.ts +3 -1
- package/src/loader/manifest-config.d.ts +18 -0
- package/src/loader/manifest-config.js +11 -3
- package/src/loader/shard-cache.d.ts +2 -0
- package/src/loader/shard-cache.js +29 -1
- package/src/models/gemma4.js +8 -0
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +159 -22
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/ffn.rules.json +33 -0
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/kernels/fused-ffn.rules.json +1 -1
- package/src/rules/kernels/lm-head-argmax.rules.json +10 -0
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/rules/kernels/silu.rules.json +1 -0
- package/src/rules/rule-registry.js +2 -0
- package/src/storage/artifact-storage-context.js +36 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-browser-command-runner.js +42 -7
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/tooling/opfs-cache.d.ts +21 -1
- package/src/tooling/opfs-cache.js +168 -32
- package/src/tooling-exports/storage.d.ts +1 -1
- package/src/tooling-exports/storage.js +1 -1
- package/src/types/gpu.d.ts +25 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +1593 -0
- package/tests/kernels/browser/test-page.js +2629 -0
- package/tests/kernels/harness/benchmark.js +225 -0
- package/tests/kernels/harness/buffer-utils.js +88 -0
- package/tests/kernels/harness/tolerance.js +174 -0
- package/tests/kernels/reference/adam.js +15 -0
- package/tests/kernels/reference/attention.js +158 -0
- package/tests/kernels/reference/backward.js +96 -0
- package/tests/kernels/reference/bias-add.js +20 -0
- package/tests/kernels/reference/conv2d.js +58 -0
- package/tests/kernels/reference/cross-entropy.js +26 -0
- package/tests/kernels/reference/dequant.js +312 -0
- package/tests/kernels/reference/embed.js +15 -0
- package/tests/kernels/reference/gather.js +61 -0
- package/tests/kernels/reference/gelu.js +37 -0
- package/tests/kernels/reference/groupnorm.js +36 -0
- package/tests/kernels/reference/index.js +89 -0
- package/tests/kernels/reference/layernorm.js +22 -0
- package/tests/kernels/reference/matmul.js +57 -0
- package/tests/kernels/reference/modulate.js +24 -0
- package/tests/kernels/reference/moe-gather.js +83 -0
- package/tests/kernels/reference/pixel-shuffle.js +20 -0
- package/tests/kernels/reference/residual.js +55 -0
- package/tests/kernels/reference/rmsnorm.js +52 -0
- package/tests/kernels/reference/rope.js +79 -0
- package/tests/kernels/reference/sample.js +97 -0
- package/tests/kernels/reference/scatter-add.js +58 -0
- package/tests/kernels/reference/silu.js +52 -0
- package/tests/kernels/reference/softmax.js +86 -0
- package/tests/kernels/reference/split-qkv.js +58 -0
- package/tests/kernels/reference/topk.js +88 -0
- package/tests/kernels/reference/upsample2d.js +37 -0
- package/BRANDING.md +0 -14
- package/SECURITY.md +0 -19
- package/assets/doppler-readme-hero.svg +0 -84
- package/assets/doppler-runtime-map.svg +0 -84
- package/src/bootstrap.js +0 -30
- package/src/cli/doppler-cli.d.ts +0 -46
- package/src/cli/doppler-quickstart.d.ts +0 -45
- package/src/cli/doppler-serve.d.ts +0 -210
- package/src/cli/runtime-profiles.d.ts +0 -40
- package/src/client/doppler-api.browser.d.ts +0 -1
- package/src/client/doppler-api.browser.js +0 -25
- package/src/client/doppler-registry.d.ts +0 -35
- package/src/client/runtime/node-quickstart-cache.d.ts +0 -22
- package/src/client/runtime/source-runtime.d.ts +0 -26
- package/src/client/runtime/storage.d.ts +0 -5
- package/src/config/README.md +0 -61
- package/src/config/conversion/README.md +0 -58
- package/src/config/goal-completion-matrix.json +0 -552
- package/src/config/kernels/kernel-ref-digests.d.ts +0 -1
- package/src/config/kernels/moe/gpt-oss.paths.json +0 -49
- package/src/config/kernels/moe/mixtral.paths.json +0 -46
- package/src/config/kernels/registry.d.ts +0 -51
- package/src/config/merge-contract-check.d.ts +0 -16
- package/src/config/merge-contract-check.js +0 -318
- package/src/config/merge-helpers.d.ts +0 -60
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +0 -66
- package/src/config/model-lanes/catalog-lane-resolver.js +0 -170
- package/src/config/param-categories.d.ts +0 -8
- package/src/config/param-validator.d.ts +0 -28
- package/src/config/platforms/loader.d.ts +0 -38
- package/src/config/quantization-contract-check.d.ts +0 -12
- package/src/config/quantization-contract-check.js +0 -91
- package/src/config/runtime/README.md +0 -28
- package/src/config/runtime-merge.d.ts +0 -5
- package/src/config/schema/claim-evidence-contract.schema.json +0 -93
- package/src/config/schema/command-surface-contract.schema.json +0 -81
- package/src/config/schema/goal-completion-matrix.schema.json +0 -223
- package/src/config/schema/model-artifact-contract.schema.json +0 -119
- package/src/config/schema/platform.schema.js +0 -1
- package/src/config/schema/policy-schema-registry.json +0 -26
- package/src/config/schema/subsystem-support-contract.schema.json +0 -48
- package/src/config/support-tiers/subsystems.json +0 -502
- package/src/config/supported-operations.d.ts +0 -14
- package/src/config/transforms/capability-transform-resolver.d.ts +0 -56
- package/src/config/transforms/execution-graph-transforms.d.ts +0 -300
- package/src/converter/conversion-plan.d.ts +0 -56
- package/src/converter/index.d.ts +0 -98
- package/src/converter/index.js +0 -59
- package/src/converter/parsers/diffusion.d.ts +0 -50
- package/src/converter/parsers/gguf.d.ts +0 -31
- package/src/converter/parsers/index.d.ts +0 -21
- package/src/converter/parsers/index.js +0 -12
- package/src/converter/parsers/transformer.d.ts +0 -31
- package/src/converter/quantization-info.d.ts +0 -38
- package/src/converter/rope-config.d.ts +0 -16
- package/src/converter/tokenizer-utils.d.ts +0 -12
- package/src/debug/reference/README.md +0 -114
- package/src/debug/reference/hf_attn_debug.py +0 -114
- package/src/debug/reference/hf_embed_check.py +0 -89
- package/src/debug/reference/hf_layer_out.py +0 -100
- package/src/debug/reference/hf_qwen35_linear_attn_debug.py +0 -268
- package/src/debug/reference/hf_rope_check.py +0 -116
- package/src/debug/reference/hf_weights.py +0 -75
- package/src/errors/doppler-error.d.ts +0 -20
- package/src/experimental/bridge/extension/background.d.ts +0 -14
- package/src/experimental/bridge/extension/background.js +0 -168
- package/src/experimental/bridge/extension/manifest.json +0 -34
- package/src/experimental/bridge/extension-client.d.ts +0 -114
- package/src/experimental/bridge/index.d.ts +0 -69
- package/src/experimental/bridge/node-bridge.js +0 -24
- package/src/experimental/bridge/protocol.d.ts +0 -96
- package/src/experimental/browser/gguf-importer.d.ts +0 -136
- package/src/experimental/browser/gguf-importer.js +0 -532
- package/src/experimental/browser/gguf-parser-browser.d.ts +0 -14
- package/src/experimental/browser/quantization.d.ts +0 -74
- package/src/experimental/browser/safetensors-parser-browser.d.ts +0 -193
- package/src/experimental/browser/shard-io-browser.d.ts +0 -57
- package/src/experimental/browser/shard-io-browser.js +0 -89
- package/src/experimental/browser/tensor-source-download.d.ts +0 -27
- package/src/experimental/diffusion/config/expansion-lanes.json +0 -29
- package/src/experimental/distribution/p2p-transport-contract.d.ts +0 -57
- package/src/experimental/distribution/p2p-transport.js +0 -75
- package/src/experimental/hotswap/intent-bundle.d.ts +0 -37
- package/src/experimental/hotswap/manifest.d.ts +0 -42
- package/src/experimental/hotswap/model-swap.js +0 -42
- package/src/experimental/hotswap/runtime.d.ts +0 -31
- package/src/experimental/training/README.md +0 -164
- package/src/experimental/training/artifacts.d.ts +0 -160
- package/src/experimental/training/attention-backward.d.ts +0 -30
- package/src/experimental/training/attention-forward.d.ts +0 -22
- package/src/experimental/training/attention-forward.js +0 -82
- package/src/experimental/training/autograd.d.ts +0 -51
- package/src/experimental/training/backward-kernels.js +0 -39
- package/src/experimental/training/checkpoint-watch.d.ts +0 -8
- package/src/experimental/training/checkpoint.d.ts +0 -36
- package/src/experimental/training/clip.d.ts +0 -9
- package/src/experimental/training/dataloader.d.ts +0 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +0 -41
- package/src/experimental/training/datasets/gepa-frontier.js +0 -220
- package/src/experimental/training/datasets/index.d.ts +0 -24
- package/src/experimental/training/datasets/index.js +0 -13
- package/src/experimental/training/datasets/jsonl.d.ts +0 -11
- package/src/experimental/training/datasets/reploid.d.ts +0 -3
- package/src/experimental/training/datasets/reploid.js +0 -36
- package/src/experimental/training/datasets/teacher-traces.d.ts +0 -51
- package/src/experimental/training/datasets/teacher-traces.js +0 -279
- package/src/experimental/training/datasets/text-pairs.d.ts +0 -66
- package/src/experimental/training/datasets/token-batch.d.ts +0 -21
- package/src/experimental/training/datasets/token-batch.js +0 -52
- package/src/experimental/training/datasets/translation-pairs.d.ts +0 -34
- package/src/experimental/training/datasets/translation-pairs.js +0 -49
- package/src/experimental/training/distillation/artifacts.d.ts +0 -71
- package/src/experimental/training/distillation/checkpoint-watch.d.ts +0 -10
- package/src/experimental/training/distillation/dataset.d.ts +0 -59
- package/src/experimental/training/distillation/eval.d.ts +0 -34
- package/src/experimental/training/distillation/index.d.ts +0 -29
- package/src/experimental/training/distillation/runtime.d.ts +0 -20
- package/src/experimental/training/distillation/scoreboard.d.ts +0 -6
- package/src/experimental/training/distillation/stage-a.d.ts +0 -45
- package/src/experimental/training/distillation/stage-b.d.ts +0 -24
- package/src/experimental/training/distillation/student-fixture.d.ts +0 -27
- package/src/experimental/training/distillation/suite-data.d.ts +0 -45
- package/src/experimental/training/export.d.ts +0 -41
- package/src/experimental/training/index.d.ts +0 -74
- package/src/experimental/training/index.js +0 -63
- package/src/experimental/training/lora-pipeline.d.ts +0 -183
- package/src/experimental/training/lora.d.ts +0 -19
- package/src/experimental/training/loss-scaling.d.ts +0 -21
- package/src/experimental/training/loss.d.ts +0 -10
- package/src/experimental/training/objectives/base.d.ts +0 -58
- package/src/experimental/training/objectives/cross_entropy.d.ts +0 -18
- package/src/experimental/training/objectives/distill_kd.d.ts +0 -16
- package/src/experimental/training/objectives/distill_triplet.d.ts +0 -16
- package/src/experimental/training/objectives/index.d.ts +0 -12
- package/src/experimental/training/objectives/index.js +0 -6
- package/src/experimental/training/objectives/ul_stage1_joint.d.ts +0 -16
- package/src/experimental/training/objectives/ul_stage2_base.d.ts +0 -16
- package/src/experimental/training/operator-agent-eval.d.ts +0 -65
- package/src/experimental/training/operator-artifacts.d.ts +0 -62
- package/src/experimental/training/operator-command.d.ts +0 -5
- package/src/experimental/training/operator-eval.d.ts +0 -48
- package/src/experimental/training/operator-scoreboard.d.ts +0 -5
- package/src/experimental/training/optimizer.d.ts +0 -22
- package/src/experimental/training/runner.d.ts +0 -249
- package/src/experimental/training/suite.d.ts +0 -299
- package/src/experimental/training/tensor-factory.d.ts +0 -9
- package/src/experimental/training/trainer.d.ts +0 -89
- package/src/experimental/training/ul_dataset.d.ts +0 -47
- package/src/experimental/training/ul_schedule.d.ts +0 -6
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +0 -175
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +0 -162
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +0 -160
- package/src/experimental/training/workload-packs/distill-translategemma-medium.json +0 -103
- package/src/experimental/training/workload-packs/distill-translategemma-tiny.json +0 -103
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +0 -152
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +0 -139
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +0 -137
- package/src/experimental/training/workload-packs/lora-toy-medium.json +0 -91
- package/src/experimental/training/workload-packs/lora-toy-tiny.json +0 -91
- package/src/experimental/training/workload-packs/registry.json +0 -278
- package/src/experimental/training/workload-packs/ul-training-medium.json +0 -55
- package/src/experimental/training/workload-packs/ul-training-tiny.json +0 -55
- package/src/experimental/training/workloads.d.ts +0 -201
- package/src/formats/gguf/types.d.ts +0 -74
- package/src/formats/index.d.ts +0 -75
- package/src/formats/index.js +0 -19
- package/src/formats/litert/types.d.ts +0 -74
- package/src/formats/rdrr/distributed/parsing.d.ts +0 -6
- package/src/formats/rdrr/distributed/validation.d.ts +0 -23
- package/src/formats/rdrr/storage-descriptor.d.ts +0 -36
- package/src/formats/rdrr/tensor-config-validator.d.ts +0 -33
- package/src/formats/safetensors/types.d.ts +0 -67
- package/src/formats/tflite/types.d.ts +0 -61
- package/src/formats/tokenizer/index.d.ts +0 -5
- package/src/formats/tokenizer/index.js +0 -3
- package/src/formats/tokenizer/types.d.ts +0 -9
- package/src/gpu/kernel-runtime.d.ts +0 -13
- package/src/gpu/kernel-selection-cache.d.ts +0 -13
- package/src/gpu/kernel-selection-log.d.ts +0 -4
- package/src/gpu/kernel-tuner/benchmarks.d.ts +0 -144
- package/src/gpu/kernel-tuner/cache.d.ts +0 -55
- package/src/gpu/kernel-tuner/index.d.ts +0 -59
- package/src/gpu/kernel-tuner/tuner.d.ts +0 -82
- package/src/gpu/kernel-tuner/types.d.ts +0 -101
- package/src/gpu/kernel-tuner/types.js +0 -4
- package/src/gpu/kernel-tuner.d.ts +0 -33
- package/src/gpu/kernels/README.md +0 -159
- package/src/gpu/kernels/check-finiteness.d.ts +0 -15
- package/src/gpu/kernels/check-hot-vocab-stop.d.ts +0 -18
- package/src/gpu/kernels/check-stop.d.ts +0 -31
- package/src/gpu/kernels/codegen/patches/attention_decode_f16.from.attention_decode.diff +0 -213
- package/src/gpu/kernels/codegen/patches/attention_decode_f16kv.from.attention_decode.diff +0 -89
- package/src/gpu/kernels/codegen/patches/attention_f16.from.attention.diff +0 -62
- package/src/gpu/kernels/codegen/patches/attention_f16kv.from.attention.diff +0 -40
- package/src/gpu/kernels/codegen/patches/attention_small_f16.from.attention_small.diff +0 -226
- package/src/gpu/kernels/codegen/patches/attention_small_f16kv.from.attention_small.diff +0 -264
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +0 -71
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16kv.from.attention_streaming.diff +0 -53
- package/src/gpu/kernels/codegen/patches/bias_add_f16.from.bias_add.diff +0 -54
- package/src/gpu/kernels/codegen/patches/dequant_mxfp4_expert_f16.from.dequant_mxfp4_expert.diff +0 -125
- package/src/gpu/kernels/codegen/patches/fused_ffn_f16.from.fused_ffn.diff +0 -428
- package/src/gpu/kernels/codegen/patches/fused_matmul_q4_batched_f16.from.fused_matmul_q4_batched.diff +0 -114
- package/src/gpu/kernels/codegen/patches/fused_matmul_rmsnorm_f16.from.fused_matmul_rmsnorm.diff +0 -477
- package/src/gpu/kernels/codegen/patches/fused_swiglu_f16.from.fused_swiglu.diff +0 -60
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -41
- package/src/gpu/kernels/codegen/patches/layernorm_f16.from.layernorm.diff +0 -82
- package/src/gpu/kernels/codegen/patches/matmul_gemv_residual_f16.from.matmul_gemv_residual.diff +0 -87
- package/src/gpu/kernels/codegen/patches/moe_gather_f16.from.moe_gather.diff +0 -181
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +0 -65
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +0 -515
- package/src/gpu/kernels/codegen/patches/rope_f16.from.rope.diff +0 -292
- package/src/gpu/kernels/codegen/patches/sample_f16.from.sample.diff +0 -336
- package/src/gpu/kernels/codegen/patches/scatter_add_dynamic_f16.from.scatter_add_dynamic.diff +0 -65
- package/src/gpu/kernels/codegen/patches/silu_f16.from.silu.diff +0 -81
- package/src/gpu/kernels/codegen/patches/topk_f16.from.topk.diff +0 -231
- package/src/gpu/kernels/codegen/wgsl-patch-variants.js +0 -152
- package/src/gpu/kernels/codegen/wgsl-variants.js +0 -885
- package/src/gpu/kernels/constants.d.ts +0 -129
- package/src/gpu/kernels/dispatch.d.ts +0 -72
- package/src/gpu/kernels/gated-short-conv.d.ts +0 -63
- package/src/gpu/kernels/kernel-base.d.ts +0 -33
- package/src/gpu/kernels/linear-attention-core.d.ts +0 -42
- package/src/gpu/kernels/matmul-dispatch.d.ts +0 -49
- package/src/gpu/kernels/matmul-selection.d.ts +0 -87
- package/src/gpu/kernels/rep-penalty.d.ts +0 -21
- package/src/gpu/kernels/rule-matcher.d.ts +0 -30
- package/src/gpu/kernels/rule-registry.d.ts +0 -7
- package/src/gpu/kernels/types.js +0 -4
- package/src/gpu/perf-guards.d.ts +0 -13
- package/src/gpu/profiler.d.ts +0 -93
- package/src/gpu/readback-utils.d.ts +0 -16
- package/src/gpu/readback-utils.js +0 -49
- package/src/gpu/submit-probe.d.ts +0 -12
- package/src/index-browser.d.ts +0 -2
- package/src/index-browser.js +0 -2
- package/src/inference/README.md +0 -593
- package/src/inference/browser-harness-contract-helpers.d.ts +0 -5
- package/src/inference/browser-harness-diffusion-energy-suites.d.ts +0 -2
- package/src/inference/browser-harness-model-helpers.d.ts +0 -10
- package/src/inference/browser-harness-report-helpers.d.ts +0 -7
- package/src/inference/browser-harness-suite-helpers.d.ts +0 -40
- package/src/inference/browser-harness-text-helpers.d.ts +0 -124
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +0 -61
- package/src/inference/model-lanes/execution-lane-resolver.js +0 -126
- package/src/inference/network-evolution.js +0 -79
- package/src/inference/pipelines/context.d.ts +0 -21
- package/src/inference/pipelines/diffusion/helpers.d.ts +0 -29
- package/src/inference/pipelines/diffusion/index.d.ts +0 -3
- package/src/inference/pipelines/diffusion/index.js +0 -3
- package/src/inference/pipelines/diffusion/scheduler.d.ts +0 -35
- package/src/inference/pipelines/diffusion/sd3-transformer.d.ts +0 -20
- package/src/inference/pipelines/diffusion/sd3-weights.d.ts +0 -21
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +0 -82
- package/src/inference/pipelines/diffusion/text-encoder.d.ts +0 -29
- package/src/inference/pipelines/diffusion/types.js +0 -1
- package/src/inference/pipelines/diffusion/vae.d.ts +0 -20
- package/src/inference/pipelines/dream/energy-head-pipeline.d.ts +0 -29
- package/src/inference/pipelines/dream/energy-head-pipeline.js +0 -6
- package/src/inference/pipelines/dream/pipeline.d.ts +0 -17
- package/src/inference/pipelines/dream/pipeline.js +0 -8
- package/src/inference/pipelines/energy/index.d.ts +0 -1
- package/src/inference/pipelines/energy/index.js +0 -1
- package/src/inference/pipelines/energy/types.js +0 -1
- package/src/inference/pipelines/energy-head/index.d.ts +0 -6
- package/src/inference/pipelines/energy-head/index.js +0 -6
- package/src/inference/pipelines/factory.d.ts +0 -10
- package/src/inference/pipelines/index.d.ts +0 -22
- package/src/inference/pipelines/index.js +0 -19
- package/src/inference/pipelines/registry.d.ts +0 -15
- package/src/inference/pipelines/rng.d.ts +0 -2
- package/src/inference/pipelines/structured/tsv-grammar-mask.js +0 -166
- package/src/inference/pipelines/text/attention/attn-config.d.ts +0 -23
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +0 -173
- package/src/inference/pipelines/text/attention/index.d.ts +0 -24
- package/src/inference/pipelines/text/attention/output-projection.d.ts +0 -12
- package/src/inference/pipelines/text/attention/precision-contract.d.ts +0 -36
- package/src/inference/pipelines/text/attention/projections.d.ts +0 -208
- package/src/inference/pipelines/text/attention/quant-options.d.ts +0 -38
- package/src/inference/pipelines/text/attention/record.d.ts +0 -36
- package/src/inference/pipelines/text/attention/rmsnorm-fusion-gate.d.ts +0 -8
- package/src/inference/pipelines/text/attention/run.d.ts +0 -38
- package/src/inference/pipelines/text/attention/types.d.ts +0 -125
- package/src/inference/pipelines/text/attention.d.ts +0 -23
- package/src/inference/pipelines/text/attention.js +0 -12
- package/src/inference/pipelines/text/bdpa-steamroller.d.ts +0 -22
- package/src/inference/pipelines/text/buffer-types.js +0 -4
- package/src/inference/pipelines/text/debug-utils/config.d.ts +0 -144
- package/src/inference/pipelines/text/debug-utils/index.d.ts +0 -53
- package/src/inference/pipelines/text/debug-utils/logging.d.ts +0 -106
- package/src/inference/pipelines/text/debug-utils/tensor.d.ts +0 -119
- package/src/inference/pipelines/text/debug-utils/utils.d.ts +0 -77
- package/src/inference/pipelines/text/debug-utils.d.ts +0 -42
- package/src/inference/pipelines/text/drift-policy.d.ts +0 -57
- package/src/inference/pipelines/text/dtype-contract.d.ts +0 -53
- package/src/inference/pipelines/text/embed.d.ts +0 -68
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +0 -49
- package/src/inference/pipelines/text/failure-reduction.d.ts +0 -84
- package/src/inference/pipelines/text/failure-reduction.js +0 -180
- package/src/inference/pipelines/text/ffn/dense.d.ts +0 -99
- package/src/inference/pipelines/text/ffn/index.d.ts +0 -23
- package/src/inference/pipelines/text/ffn/moe.d.ts +0 -24
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +0 -27
- package/src/inference/pipelines/text/ffn/standard.d.ts +0 -25
- package/src/inference/pipelines/text/ffn/types.d.ts +0 -30
- package/src/inference/pipelines/text/ffn.d.ts +0 -31
- package/src/inference/pipelines/text/ffn.js +0 -18
- package/src/inference/pipelines/text/finiteness-guard-status.d.ts +0 -11
- package/src/inference/pipelines/text/finiteness-policy.d.ts +0 -39
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +0 -81
- package/src/inference/pipelines/text/generator-helpers.d.ts +0 -45
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +0 -84
- package/src/inference/pipelines/text/generator-runtime.d.ts +0 -89
- package/src/inference/pipelines/text/generator-steps.d.ts +0 -186
- package/src/inference/pipelines/text/index.d.ts +0 -5
- package/src/inference/pipelines/text/index.js +0 -6
- package/src/inference/pipelines/text/kernel-trace.d.ts +0 -119
- package/src/inference/pipelines/text/layer.d.ts +0 -118
- package/src/inference/pipelines/text/load-errors.d.ts +0 -21
- package/src/inference/pipelines/text/logits/types.js +0 -4
- package/src/inference/pipelines/text/logits.d.ts +0 -27
- package/src/inference/pipelines/text/logits.js +0 -16
- package/src/inference/pipelines/text/lora-apply.d.ts +0 -28
- package/src/inference/pipelines/text/moe-cache.d.ts +0 -32
- package/src/inference/pipelines/text/moe-cpu-gptoss.d.ts +0 -9
- package/src/inference/pipelines/text/moe-cpu.d.ts +0 -13
- package/src/inference/pipelines/text/moe-gpu.d.ts +0 -26
- package/src/inference/pipelines/text/moe-helpers.d.ts +0 -12
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +0 -40
- package/src/inference/pipelines/text/operator-events.d.ts +0 -125
- package/src/inference/pipelines/text/operator-identity.d.ts +0 -60
- package/src/inference/pipelines/text/ops.d.ts +0 -219
- package/src/inference/pipelines/text/per-layer-inputs.d.ts +0 -107
- package/src/inference/pipelines/text/probes.d.ts +0 -28
- package/src/inference/pipelines/text/sampling-config.d.ts +0 -48
- package/src/inference/pipelines/text/stage-names.d.ts +0 -88
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +0 -57
- package/src/inference/pipelines/text/types.js +0 -4
- package/src/inference/tokenizers/base.d.ts +0 -44
- package/src/inference/tokenizers/behavior-flags.d.ts +0 -9
- package/src/inference/tokenizers/bpe.d.ts +0 -27
- package/src/inference/tokenizers/bundled.d.ts +0 -67
- package/src/inference/tokenizers/sentencepiece.d.ts +0 -30
- package/src/inference/tokenizers/types.js +0 -7
- package/src/loader/embedding-limit-preflight.d.ts +0 -31
- package/src/loader/per-layer-input-loader.d.ts +0 -30
- package/src/loader/tensors/functional-descriptor-loader.d.ts +0 -9
- package/src/loader/tensors/source-transform.d.ts +0 -65
- package/src/memory/address-table.d.ts +0 -96
- package/src/rules/execution-rules-contract-check.d.ts +0 -21
- package/src/rules/layer-pattern-contract-check.d.ts +0 -17
- package/src/rules/rule-registry.d.ts +0 -77
- package/src/storage/artifact-storage-context.d.ts +0 -63
- package/src/storage/backends/idb-store.d.ts +0 -52
- package/src/storage/backends/memory-store.d.ts +0 -36
- package/src/storage/backends/opfs-store.d.ts +0 -41
- package/src/storage/blake3.d.ts +0 -17
- package/src/storage/download-types.d.ts +0 -157
- package/src/storage/downloader.d.ts +0 -103
- package/src/storage/index.d.ts +0 -274
- package/src/storage/index.js +0 -201
- package/src/storage/preflight.d.ts +0 -144
- package/src/storage/preflight.js +0 -360
- package/src/storage/quickstart-downloader.d.ts +0 -157
- package/src/storage/quickstart-downloader.js +0 -289
- package/src/storage/source-artifact-store.d.ts +0 -59
- package/src/sw.js +0 -184
- package/src/tooling/command-api-constants.d.ts +0 -9
- package/src/tooling/command-api-family-normalizers.d.ts +0 -10
- package/src/tooling/command-api-helpers.d.ts +0 -23
- package/src/tooling/command-runner-shared.d.ts +0 -89
- package/src/tooling/conversion-config-materializer.d.ts +0 -14
- package/src/tooling/conversion-config-materializer.js +0 -96
- package/src/tooling/diagnose-runner.d.ts +0 -28
- package/src/tooling/hf-registry-utils.d.ts +0 -55
- package/src/tooling/kernel-path-builder/custom-runtime-facts.d.ts +0 -18
- package/src/tooling/kernel-path-builder/custom-runtime-facts.js +0 -72
- package/src/tooling/kernel-path-builder/index.d.ts +0 -65
- package/src/tooling/kernel-path-builder/index.js +0 -1361
- package/src/tooling/kernel-path-builder/runtime-overlay.d.ts +0 -9
- package/src/tooling/kernel-path-builder/runtime-overlay.js +0 -147
- package/src/tooling/lean-execution-contract-runner.d.ts +0 -33
- package/src/tooling/lean-execution-contract-runner.js +0 -158
- package/src/tooling/lean-execution-contract.d.ts +0 -16
- package/src/tooling/lean-execution-contract.js +0 -81
- package/src/tooling/litert-package-runtime.d.ts +0 -46
- package/src/tooling/node-convert-worker-pool.d.ts +0 -16
- package/src/tooling/node-convert-worker.d.ts +0 -1
- package/src/tooling/node-converter.d.ts +0 -1
- package/src/tooling/node-file-fetch.d.ts +0 -1
- package/src/tooling/node-source-runtime.d.ts +0 -24
- package/src/tooling/node-webgpu.d.ts +0 -17
- package/src/tooling/precision-replay-math.js +0 -174
- package/src/tooling/program-bundle-host.d.ts +0 -16
- package/src/tooling/program-bundle-host.js +0 -20
- package/src/tooling/runtime-input-composition.d.ts +0 -37
- package/src/tooling/source-package-profiles.d.ts +0 -50
- package/src/tooling/source-runtime-converter-config.d.ts +0 -19
- package/src/tooling/source-runtime-materializer.d.ts +0 -6
- package/src/tooling/source-runtime-materializer.js +0 -89
- package/src/tooling-exports.browser.d.ts +0 -7
- package/src/types/chrome.d.ts +0 -36
- package/src/types/chrome.js +0 -1
- package/src/types/index.d.ts +0 -1
- package/src/types/index.js +0 -1
- package/src/utils/array-buffer.d.ts +0 -4
- package/src/utils/canonical-hash.d.ts +0 -7
- package/src/utils/clone-json.d.ts +0 -1
- package/src/utils/hf-resolve-url.d.ts +0 -16
- package/src/utils/index.d.ts +0 -10
- package/src/utils/index.js +0 -10
- package/src/utils/load-json.d.ts +0 -5
- package/src/utils/plain-object.d.ts +0 -1
- package/src/utils/sha256.d.ts +0 -5
- package/src/utils/stable-sort-object.d.ts +0 -1
|
@@ -245,6 +245,23 @@ export function buildConservativeMultimodalGenerationOptions(options = {}) {
|
|
|
245
245
|
};
|
|
246
246
|
}
|
|
247
247
|
|
|
248
|
+
function resolveMultimodalMaxTokens(runtimeConfig, requestedMaxTokens) {
|
|
249
|
+
if (requestedMaxTokens !== undefined) {
|
|
250
|
+
return requirePositiveInteger(requestedMaxTokens, 'maxTokens');
|
|
251
|
+
}
|
|
252
|
+
return requirePositiveInteger(
|
|
253
|
+
runtimeConfig?.inference?.generation?.multimodalMaxTokens,
|
|
254
|
+
'runtime.inference.generation.multimodalMaxTokens'
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function requirePositiveInteger(value, label) {
|
|
259
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
260
|
+
throw new Error(`${label} must be a positive integer.`);
|
|
261
|
+
}
|
|
262
|
+
return value;
|
|
263
|
+
}
|
|
264
|
+
|
|
248
265
|
// ============================================================================
|
|
249
266
|
// Main Inference Pipeline Class
|
|
250
267
|
// ============================================================================
|
|
@@ -282,6 +299,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
282
299
|
assignStorageContext: true,
|
|
283
300
|
});
|
|
284
301
|
this.runtimeConfig = runtimeConfig;
|
|
302
|
+
this.dopplerLoader = contexts.loader || null;
|
|
303
|
+
this.ownsDopplerLoader = contexts.ownsLoader === true;
|
|
285
304
|
this.runtimeOverrides = contexts.runtimeConfig == null
|
|
286
305
|
? null
|
|
287
306
|
: (typeof structuredClone === 'function'
|
|
@@ -383,6 +402,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
383
402
|
numLayers: Number(manifest.architecture?.numLayers ?? 0),
|
|
384
403
|
capabilities,
|
|
385
404
|
platform,
|
|
405
|
+
useGPU: this.useGPU === true,
|
|
386
406
|
});
|
|
387
407
|
if (executionV1Runtime.executionV1State) {
|
|
388
408
|
this.runtimeConfig = executionV1Runtime.runtimeConfig;
|
|
@@ -631,6 +651,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
631
651
|
this.modelConfig.perLayerInputsSession ?? null,
|
|
632
652
|
this.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
633
653
|
),
|
|
654
|
+
loader: this.dopplerLoader ?? undefined,
|
|
634
655
|
onProgress: (info) => {
|
|
635
656
|
if (info.stage !== 'layers' && info.stage !== 'shards') {
|
|
636
657
|
log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%${info.message ? ` - ${info.message}` : ''}`);
|
|
@@ -658,7 +679,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
658
679
|
|
|
659
680
|
this.layerRouterWeights = result.layerRouterWeights;
|
|
660
681
|
|
|
661
|
-
this.dopplerLoader = getDopplerLoader(this.runtimeConfig.loading);
|
|
682
|
+
this.dopplerLoader = result.loader ?? getDopplerLoader(this.runtimeConfig.loading);
|
|
662
683
|
this.stats.loadTiming = result.loadTiming ?? this.dopplerLoader?.getLoadTiming?.() ?? null;
|
|
663
684
|
|
|
664
685
|
if ((this.modelConfig).useMoE && this.moeRouter) {
|
|
@@ -670,7 +691,11 @@ export class InferencePipeline extends PipelineState {
|
|
|
670
691
|
}
|
|
671
692
|
|
|
672
693
|
if (this.useGPU && this.modelConfig) {
|
|
673
|
-
|
|
694
|
+
const session = this.runtimeConfig?.inference?.session ?? null;
|
|
695
|
+
fuseQKVWeights(result.layerWeights, this.modelConfig, this.resolvedKernelPath, {
|
|
696
|
+
allowQ4K: session?.useFusedQKVSplitQKNorm === true
|
|
697
|
+
|| session?.useFusedQKVSplitQKNormRoPE === true,
|
|
698
|
+
});
|
|
674
699
|
}
|
|
675
700
|
|
|
676
701
|
if (this.useGPU && this.modelConfig) {
|
|
@@ -1145,7 +1170,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
1145
1170
|
|
|
1146
1171
|
// Step 3: Generate with embedding override at the image token offset.
|
|
1147
1172
|
const tokens = [];
|
|
1148
|
-
const maxGen = maxTokens
|
|
1173
|
+
const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
|
|
1149
1174
|
const stopTokenIds = this.modelConfig.stopTokenIds;
|
|
1150
1175
|
|
|
1151
1176
|
try {
|
|
@@ -1274,7 +1299,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
1274
1299
|
|
|
1275
1300
|
// Step 3: Generate with embedding override at the video token offset
|
|
1276
1301
|
const tokens = [];
|
|
1277
|
-
const maxGen = maxTokens
|
|
1302
|
+
const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
|
|
1278
1303
|
const stopTokenIds = this.modelConfig.stopTokenIds;
|
|
1279
1304
|
|
|
1280
1305
|
try {
|
|
@@ -1404,7 +1429,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
1404
1429
|
|
|
1405
1430
|
// Step 4: Generate with embedding override at the audio token offset
|
|
1406
1431
|
const tokens = [];
|
|
1407
|
-
const maxGen = maxTokens
|
|
1432
|
+
const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
|
|
1408
1433
|
const stopTokenIds = this.modelConfig.stopTokenIds;
|
|
1409
1434
|
|
|
1410
1435
|
try {
|
|
@@ -1560,14 +1585,23 @@ export class InferencePipeline extends PipelineState {
|
|
|
1560
1585
|
|
|
1561
1586
|
async embed(prompt, options = {}) {
|
|
1562
1587
|
assertNotAborted(options?.signal);
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1588
|
+
this.resetForBatch();
|
|
1589
|
+
try {
|
|
1590
|
+
const result = await this.prefillWithEmbedding(prompt, {
|
|
1591
|
+
...options,
|
|
1592
|
+
__skipStateSnapshot: true,
|
|
1593
|
+
});
|
|
1594
|
+
assertNotAborted(options?.signal);
|
|
1595
|
+
return {
|
|
1596
|
+
embedding: result.embedding,
|
|
1597
|
+
tokens: result.tokens,
|
|
1598
|
+
seqLen: result.seqLen,
|
|
1599
|
+
embeddingMode: result.embeddingMode,
|
|
1600
|
+
phase: result.phase ?? null,
|
|
1601
|
+
};
|
|
1602
|
+
} finally {
|
|
1603
|
+
this.resetForBatch();
|
|
1604
|
+
}
|
|
1571
1605
|
}
|
|
1572
1606
|
|
|
1573
1607
|
async embedBatch(prompts, options = {}) {
|
|
@@ -1581,7 +1615,6 @@ export class InferencePipeline extends PipelineState {
|
|
|
1581
1615
|
// Check between every prompt so a superseded revision drops the rest.
|
|
1582
1616
|
assertNotAborted(options?.signal);
|
|
1583
1617
|
outputs.push(await this.embed(prompt, batchOptions));
|
|
1584
|
-
this.resetForBatch();
|
|
1585
1618
|
}
|
|
1586
1619
|
return outputs;
|
|
1587
1620
|
}
|
|
@@ -1726,6 +1759,14 @@ export class InferencePipeline extends PipelineState {
|
|
|
1726
1759
|
return this.generator.prefillWithLogits(prompt, options);
|
|
1727
1760
|
}
|
|
1728
1761
|
|
|
1762
|
+
prefillWithTokenLogits(prompt, tokenIds, options = {}) {
|
|
1763
|
+
return this.generator.prefillWithTokenLogits(prompt, tokenIds, options);
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options = {}) {
|
|
1767
|
+
return this.generator.prefillWithTokenLogitsFromKV(prefix, prompt, tokenIds, options);
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1729
1770
|
|
|
1730
1771
|
applyKVCacheSnapshot(snapshot) {
|
|
1731
1772
|
this.kvCache = snapshot.cache.clone();
|
|
@@ -1883,6 +1924,11 @@ export class InferencePipeline extends PipelineState {
|
|
|
1883
1924
|
this.plePrefetchPending = null;
|
|
1884
1925
|
this.weights.clear();
|
|
1885
1926
|
this.expertWeights.clear();
|
|
1927
|
+
if (this.ownsDopplerLoader && this.dopplerLoader) {
|
|
1928
|
+
await this.dopplerLoader.unload();
|
|
1929
|
+
}
|
|
1930
|
+
this.dopplerLoader = null;
|
|
1931
|
+
this.ownsDopplerLoader = false;
|
|
1886
1932
|
this.linearAttentionRuntime = resetLinearAttentionRuntime(this.linearAttentionRuntime);
|
|
1887
1933
|
this.lora = null;
|
|
1888
1934
|
destroyMoERouter(this.moeRouter);
|
|
@@ -12,7 +12,10 @@ function isBosLikeLabel(value) {
|
|
|
12
12
|
|
|
13
13
|
function isEosLikeLabel(value) {
|
|
14
14
|
const normalized = normalizeSpecialTokenLabel(value);
|
|
15
|
-
return normalized === '<eos>'
|
|
15
|
+
return normalized === '<eos>'
|
|
16
|
+
|| normalized === '</s>'
|
|
17
|
+
|| normalized.includes('eos')
|
|
18
|
+
|| normalized.includes('endoftext');
|
|
16
19
|
}
|
|
17
20
|
|
|
18
21
|
function classifySpecialTokenReference(ref, specialTokens) {
|
|
@@ -84,7 +87,25 @@ export function inferBundledTokenizerBehaviorFlags(tokenizerJson, specialTokens
|
|
|
84
87
|
return { addBosToken: null, addEosToken: null };
|
|
85
88
|
}
|
|
86
89
|
|
|
87
|
-
const
|
|
90
|
+
const findTemplateProcessing = (postProcessor) => {
|
|
91
|
+
if (!postProcessor || typeof postProcessor !== 'object' || Array.isArray(postProcessor)) {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
if (postProcessor.type === 'TemplateProcessing') {
|
|
95
|
+
return postProcessor;
|
|
96
|
+
}
|
|
97
|
+
if (postProcessor.type === 'Sequence' && Array.isArray(postProcessor.processors)) {
|
|
98
|
+
for (const child of postProcessor.processors) {
|
|
99
|
+
const template = findTemplateProcessing(child);
|
|
100
|
+
if (template) {
|
|
101
|
+
return template;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const postProcessor = findTemplateProcessing(tokenizerJson.post_processor);
|
|
88
109
|
if (!postProcessor || postProcessor.type !== 'TemplateProcessing') {
|
|
89
110
|
return { addBosToken: null, addEosToken: null };
|
|
90
111
|
}
|
|
@@ -118,6 +118,28 @@ function parseByteTokenValue(token) {
|
|
|
118
118
|
return hi >= 0 && lo >= 0 ? (hi << 4) | lo : null;
|
|
119
119
|
}
|
|
120
120
|
|
|
121
|
+
function decodeByteFallbackTokens(tokens) {
|
|
122
|
+
const parts = [];
|
|
123
|
+
let bytes = [];
|
|
124
|
+
const flushBytes = () => {
|
|
125
|
+
if (bytes.length === 0) return;
|
|
126
|
+
parts.push(new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes)));
|
|
127
|
+
bytes = [];
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
for (const token of tokens) {
|
|
131
|
+
const byteValue = parseByteTokenValue(token);
|
|
132
|
+
if (byteValue !== null) {
|
|
133
|
+
bytes.push(byteValue);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
flushBytes();
|
|
137
|
+
parts.push(token);
|
|
138
|
+
}
|
|
139
|
+
flushBytes();
|
|
140
|
+
return parts.join('');
|
|
141
|
+
}
|
|
142
|
+
|
|
121
143
|
function appendVocabEntry(token, id, vocab, reverseVocab, byteTokens) {
|
|
122
144
|
const numId = typeof id === 'number' ? id : parseInt( (id), 10);
|
|
123
145
|
vocab.set(token, numId);
|
|
@@ -469,9 +491,6 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
469
491
|
}
|
|
470
492
|
log.info('Tokenizer', `HuggingFace model.type="${model.type}", using type="${this.#type}"`);
|
|
471
493
|
this.#byteDecoder = null;
|
|
472
|
-
if (this.#type === 'bpe') {
|
|
473
|
-
this.#initializeByteDecoder();
|
|
474
|
-
}
|
|
475
494
|
let maxId = -1;
|
|
476
495
|
|
|
477
496
|
// Handle vocab based on type
|
|
@@ -586,6 +605,9 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
586
605
|
const decoderPrepend = hf.decoder?.prepend_scheme === 'always' || hf.decoder?.add_prefix_space === true;
|
|
587
606
|
const normalizerPrepend = hf.normalizer?.prepend_scheme === 'always' || hf.normalizer?.add_prefix_space === true;
|
|
588
607
|
this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
|
|
608
|
+
if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
|
|
609
|
+
this.#initializeByteDecoder();
|
|
610
|
+
}
|
|
589
611
|
const runtimeSpacePrefix = runtimeDefaults.addSpacePrefix;
|
|
590
612
|
// Use explicit runtime config if set (non-null), otherwise auto-detect from tokenizer.json
|
|
591
613
|
this.#addSpacePrefix = runtimeSpacePrefix
|
|
@@ -646,9 +668,6 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
646
668
|
throw new Error(`[Tokenizer] Unsupported tokenizer type: ${tokenizerJson.type}`);
|
|
647
669
|
}
|
|
648
670
|
this.#byteDecoder = null;
|
|
649
|
-
if (this.#type === 'bpe') {
|
|
650
|
-
this.#initializeByteDecoder();
|
|
651
|
-
}
|
|
652
671
|
|
|
653
672
|
// Build vocab maps
|
|
654
673
|
let maxId = loadObjectVocab(tokenizerJson.vocab, this.#vocab, this.#reverseVocab, this.#byteTokens);
|
|
@@ -744,6 +763,9 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
744
763
|
throw new Error('[Tokenizer] addEosToken is enabled but eos token is missing.');
|
|
745
764
|
}
|
|
746
765
|
this.#useByteLevelEncoding = byteLevelPretokenizer.useByteLevel;
|
|
766
|
+
if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
|
|
767
|
+
this.#initializeByteDecoder();
|
|
768
|
+
}
|
|
747
769
|
// NOTE: Default to FALSE - first word shouldn't get space prefix
|
|
748
770
|
// Space prefixes are only for words that follow a space in original text
|
|
749
771
|
this.#addSpacePrefix = tokenizerJson.addSpacePrefix === true
|
|
@@ -1073,29 +1095,29 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
1073
1095
|
|
|
1074
1096
|
const token = this.#reverseVocab.get(id);
|
|
1075
1097
|
if (token !== undefined) {
|
|
1076
|
-
|
|
1077
|
-
if (token.match(/^<0x[0-9A-Fa-f]{2}>$/)) {
|
|
1078
|
-
const byteVal = parseInt(token.slice(3, 5), 16);
|
|
1079
|
-
tokens.push(String.fromCharCode(byteVal));
|
|
1080
|
-
} else {
|
|
1081
|
-
tokens.push(token);
|
|
1082
|
-
}
|
|
1098
|
+
tokens.push(token);
|
|
1083
1099
|
}
|
|
1084
1100
|
}
|
|
1085
1101
|
|
|
1086
1102
|
let result;
|
|
1087
|
-
if (this.#type === 'bpe' && this.#
|
|
1088
|
-
const merged = tokens.join('');
|
|
1103
|
+
if (this.#type === 'bpe' && this.#useByteLevelEncoding) {
|
|
1089
1104
|
const bytes = [];
|
|
1090
|
-
for (const
|
|
1091
|
-
const
|
|
1092
|
-
if (
|
|
1093
|
-
bytes.push(
|
|
1105
|
+
for (const token of tokens) {
|
|
1106
|
+
const byteValue = parseByteTokenValue(token);
|
|
1107
|
+
if (byteValue !== null) {
|
|
1108
|
+
bytes.push(byteValue);
|
|
1094
1109
|
continue;
|
|
1095
1110
|
}
|
|
1096
|
-
const
|
|
1097
|
-
|
|
1098
|
-
|
|
1111
|
+
for (const ch of token) {
|
|
1112
|
+
const mapped = this.#byteDecoder.get(ch);
|
|
1113
|
+
if (mapped != null) {
|
|
1114
|
+
bytes.push(mapped);
|
|
1115
|
+
continue;
|
|
1116
|
+
}
|
|
1117
|
+
const fallbackBytes = new TextEncoder().encode(ch);
|
|
1118
|
+
for (const byte of fallbackBytes) {
|
|
1119
|
+
bytes.push(byte);
|
|
1120
|
+
}
|
|
1099
1121
|
}
|
|
1100
1122
|
}
|
|
1101
1123
|
result = new TextDecoder('utf-8', { fatal: false }).decode(Uint8Array.from(bytes));
|
|
@@ -1103,7 +1125,7 @@ export class BundledTokenizer extends BaseTokenizer {
|
|
|
1103
1125
|
result = result.replace(/▁/g, ' ');
|
|
1104
1126
|
} else {
|
|
1105
1127
|
// Join and convert ▁ back to spaces, handle GPT-style markers
|
|
1106
|
-
result = tokens
|
|
1128
|
+
result = decodeByteFallbackTokens(tokens)
|
|
1107
1129
|
.replace(/▁/g, ' ')
|
|
1108
1130
|
.replace(/Ġ/g, ' ')
|
|
1109
1131
|
.replace(/Ċ/g, '\n');
|
|
@@ -4,9 +4,13 @@
|
|
|
4
4
|
* @module inference/tokenizers/types
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
export interface SpecialTokens {
|
|
8
|
+
bos?: number;
|
|
9
|
+
eos?: number;
|
|
10
|
+
pad?: number;
|
|
11
|
+
unk?: number;
|
|
12
|
+
mask?: number;
|
|
13
|
+
}
|
|
10
14
|
|
|
11
15
|
/** Tokenizer Configuration */
|
|
12
16
|
export interface TokenizerConfig {
|
|
@@ -144,4 +148,3 @@ export interface BundledTokenizerJson {
|
|
|
144
148
|
addEosToken?: boolean;
|
|
145
149
|
addSpacePrefix?: boolean;
|
|
146
150
|
}
|
|
147
|
-
|
|
@@ -35,6 +35,7 @@ import { buildTensorLocations } from './shard-resolver.js';
|
|
|
35
35
|
import {
|
|
36
36
|
needsNormWeightOffset,
|
|
37
37
|
resolveWeightLayout,
|
|
38
|
+
requiresCpuF16ToF32MatmulMaterialization,
|
|
38
39
|
shouldStreamLargeWeight,
|
|
39
40
|
} from './manifest-config.js';
|
|
40
41
|
import { MemoryMonitor } from './memory-monitor.js';
|
|
@@ -119,7 +120,7 @@ function createLoadTiming(modelId, hasCustomLoader) {
|
|
|
119
120
|
status: 'running',
|
|
120
121
|
customShardLoader: hasCustomLoader === true,
|
|
121
122
|
byteAccountingMode: hasCustomLoader === true
|
|
122
|
-
? 'custom-loader-progress
|
|
123
|
+
? 'custom-loader-read-progress'
|
|
123
124
|
: 'full-shard-progress',
|
|
124
125
|
totalBytes: null,
|
|
125
126
|
totalShards: null,
|
|
@@ -588,10 +589,21 @@ export class DopplerLoader {
|
|
|
588
589
|
const loadStartTime = Date.now();
|
|
589
590
|
let bytesLoaded = 0;
|
|
590
591
|
let shardsLoaded = 0;
|
|
592
|
+
this.shardCache.resetCustomReadStats();
|
|
593
|
+
|
|
594
|
+
const syncCustomReadStats = () => {
|
|
595
|
+
if (!this.shardCache.hasCustomLoader) return;
|
|
596
|
+
const stats = this.shardCache.customReadStats;
|
|
597
|
+
bytesLoaded = stats.bytesRead;
|
|
598
|
+
shardsLoaded = stats.shardsRead;
|
|
599
|
+
this.loadTiming.bytesLoaded = bytesLoaded;
|
|
600
|
+
this.loadTiming.shardsLoaded = shardsLoaded;
|
|
601
|
+
};
|
|
591
602
|
|
|
592
603
|
|
|
593
604
|
const reportProgress = (stage, baseProgress, detail) => {
|
|
594
605
|
if (!onProgress || typeof onProgress !== 'function') return;
|
|
606
|
+
syncCustomReadStats();
|
|
595
607
|
const elapsed = (Date.now() - loadStartTime) / 1000;
|
|
596
608
|
const speed = elapsed > 0 ? bytesLoaded / elapsed : 0;
|
|
597
609
|
const speedStr = speed > 0 ? `${formatBytes(speed)}/s` : '';
|
|
@@ -750,6 +762,7 @@ export class DopplerLoader {
|
|
|
750
762
|
}
|
|
751
763
|
|
|
752
764
|
if (onProgress) {
|
|
765
|
+
syncCustomReadStats();
|
|
753
766
|
const layerFraction = (l + 1) / numLayers;
|
|
754
767
|
const layerProgress = 0.80 + layerFraction * 0.05;
|
|
755
768
|
onProgress({
|
|
@@ -785,9 +798,17 @@ export class DopplerLoader {
|
|
|
785
798
|
await this.#loadFinalWeights(onProgress);
|
|
786
799
|
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
787
800
|
this.#assertResidentBudget('final weights');
|
|
801
|
+
syncCustomReadStats();
|
|
788
802
|
|
|
789
803
|
if (onProgress) {
|
|
790
|
-
onProgress({
|
|
804
|
+
onProgress({
|
|
805
|
+
stage: 'complete',
|
|
806
|
+
progress: 1.0,
|
|
807
|
+
shard: shardsLoaded,
|
|
808
|
+
totalShards,
|
|
809
|
+
bytesLoaded,
|
|
810
|
+
totalBytes,
|
|
811
|
+
});
|
|
791
812
|
}
|
|
792
813
|
|
|
793
814
|
this.isLoaded = true;
|
|
@@ -804,6 +825,7 @@ export class DopplerLoader {
|
|
|
804
825
|
return (this.manifest.config) || {};
|
|
805
826
|
} catch (error) {
|
|
806
827
|
loadError = error;
|
|
828
|
+
syncCustomReadStats();
|
|
807
829
|
finishLoadTiming(this.loadTiming, 'failed', loadTimingStart, error, activeLoadPhase);
|
|
808
830
|
} finally {
|
|
809
831
|
this.#loadShardOverride = null;
|
|
@@ -1157,6 +1179,7 @@ export class DopplerLoader {
|
|
|
1157
1179
|
if (!location?.size || location.size <= 0) return false;
|
|
1158
1180
|
if (hasSourceTransform(location)) return false;
|
|
1159
1181
|
if (Array.isArray(location?.storage?.companions) && location.storage.companions.length > 0) return false;
|
|
1182
|
+
if (requiresCpuF16ToF32MatmulMaterialization(location, this.gpuCapabilities, this.keepF32Weights)) return false;
|
|
1160
1183
|
if (this.shardCache.hasCustomLoader && !this.shardCache.canStreamRanges) return false;
|
|
1161
1184
|
const chunkBytes = this.#loadingConfig?.storage?.backend?.streaming?.readChunkBytes ?? 0;
|
|
1162
1185
|
if (!Number.isFinite(chunkBytes) || chunkBytes <= 0) return false;
|
|
@@ -167,6 +167,16 @@ async function loadFirstExistingTensor(ctx, prefixes, suffixes, options = {}) {
|
|
|
167
167
|
return null;
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
+
function wrapFinalNormTensor(ctx, tensor, name, location) {
|
|
171
|
+
if (!isGpuBufferInstance(tensor) || !location?.shape) {
|
|
172
|
+
return tensor;
|
|
173
|
+
}
|
|
174
|
+
const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
175
|
+
locationDtype: location.dtype,
|
|
176
|
+
});
|
|
177
|
+
return createWeightBuffer(tensor, dtype, 'row', location.shape, name);
|
|
178
|
+
}
|
|
179
|
+
|
|
170
180
|
async function loadDiffusionGemmaSelfConditioning(ctx) {
|
|
171
181
|
const preNorm = await loadFirstExistingTensor(
|
|
172
182
|
ctx,
|
|
@@ -284,7 +294,8 @@ async function loadFinalNorm(ctx) {
|
|
|
284
294
|
for (const name of finalNormNames) {
|
|
285
295
|
const location = ctx.tensorLocations.get(name);
|
|
286
296
|
if (location) {
|
|
287
|
-
|
|
297
|
+
const tensor = await ctx.loadTensor(name, true, true);
|
|
298
|
+
finalNorm = wrapFinalNormTensor(ctx, tensor, name, location);
|
|
288
299
|
break;
|
|
289
300
|
}
|
|
290
301
|
}
|
package/src/loader/index.d.ts
CHANGED
|
@@ -42,11 +42,13 @@ export {
|
|
|
42
42
|
getLargeWeightConfig,
|
|
43
43
|
getLargeWeightMaxBytes,
|
|
44
44
|
estimateMatmulWeightBytes,
|
|
45
|
+
resolveMatmulWeightDtype,
|
|
46
|
+
requiresCpuF16ToF32MatmulMaterialization,
|
|
45
47
|
resolveWeightLayout,
|
|
46
48
|
shouldStreamLargeWeight,
|
|
47
49
|
isMoEModel,
|
|
48
50
|
} from './manifest-config.js';
|
|
49
|
-
export type { LargeWeightConfig } from '
|
|
51
|
+
export type { LargeWeightConfigSchema as LargeWeightConfig } from '../config/schema/doppler.schema.js';
|
|
50
52
|
|
|
51
53
|
export { buildTensorLocations } from './shard-resolver.js';
|
|
52
54
|
export type { BuildTensorLocationsOptions } from './shard-resolver.js';
|
package/src/loader/index.js
CHANGED
|
@@ -45,6 +45,8 @@ export interface LayerWeights {
|
|
|
45
45
|
qkvProj?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
46
46
|
qkvSizes?: [number, number, number] | null;
|
|
47
47
|
qkvDtype?: 'f16' | 'f32' | null;
|
|
48
|
+
linearABProj?: WeightBuffer | null;
|
|
49
|
+
linearQKVZProj?: WeightBuffer | null;
|
|
48
50
|
linearInProjZ?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
49
51
|
linearInProjA?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
50
52
|
linearInProjB?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
@@ -158,7 +160,7 @@ export interface LoaderLoadTiming {
|
|
|
158
160
|
modelId: string | null;
|
|
159
161
|
status: 'running' | 'complete' | 'failed';
|
|
160
162
|
customShardLoader: boolean;
|
|
161
|
-
byteAccountingMode: 'full-shard-progress' | 'custom-loader-progress
|
|
163
|
+
byteAccountingMode: 'full-shard-progress' | 'custom-loader-read-progress';
|
|
162
164
|
totalBytes: number | null;
|
|
163
165
|
totalShards: number | null;
|
|
164
166
|
bytesLoaded: number;
|
|
@@ -59,6 +59,24 @@ export declare function estimateMatmulWeightBytes(
|
|
|
59
59
|
keepF32Weights: boolean
|
|
60
60
|
): { bytes: number; dtype: WeightDtype } | null;
|
|
61
61
|
|
|
62
|
+
/**
|
|
63
|
+
* Resolve the materialized dtype for a weight-like tensor.
|
|
64
|
+
*/
|
|
65
|
+
export declare function resolveMatmulWeightDtype(
|
|
66
|
+
location: TensorLocation,
|
|
67
|
+
gpuCapabilities: KernelCapabilities | null,
|
|
68
|
+
keepF32Weights: boolean
|
|
69
|
+
): WeightDtype;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Check whether an F16 tensor must be widened on CPU before GPU upload.
|
|
73
|
+
*/
|
|
74
|
+
export declare function requiresCpuF16ToF32MatmulMaterialization(
|
|
75
|
+
location: TensorLocation,
|
|
76
|
+
gpuCapabilities: KernelCapabilities | null,
|
|
77
|
+
keepF32Weights: boolean
|
|
78
|
+
): boolean;
|
|
79
|
+
|
|
62
80
|
/**
|
|
63
81
|
* Resolve weight layout from tensor location.
|
|
64
82
|
*
|
|
@@ -67,19 +67,27 @@ export function estimateMatmulWeightBytes(location, gpuCapabilities, keepF32Weig
|
|
|
67
67
|
const numElements = location.shape.reduce((a, b) => a * b, 1);
|
|
68
68
|
if (!Number.isFinite(numElements) || numElements <= 0) return null;
|
|
69
69
|
|
|
70
|
+
const dtype = resolveMatmulWeightDtype(location, gpuCapabilities, keepF32Weights);
|
|
71
|
+
const bytesPerElement = DTYPE_SIZES[selectRuleValue('shared', 'dtype', 'f16OrF32FromDtype', { dtype })];
|
|
72
|
+
return { bytes: numElements * bytesPerElement, dtype };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function resolveMatmulWeightDtype(location, gpuCapabilities, keepF32Weights) {
|
|
70
76
|
const caps = gpuCapabilities || getKernelCapabilities();
|
|
71
77
|
const hasF16 = caps?.hasF16 ?? false;
|
|
72
78
|
const isMatmulWeight = shouldDequantizeToF16(location);
|
|
73
79
|
|
|
74
|
-
|
|
80
|
+
return selectRuleValue('loader', 'weights', 'matmulWeightDtype', {
|
|
75
81
|
locationDtype: location.dtype,
|
|
76
82
|
hasF16,
|
|
77
83
|
isMatmulWeight,
|
|
78
84
|
keepF32Weights: Boolean(keepF32Weights),
|
|
79
85
|
});
|
|
86
|
+
}
|
|
80
87
|
|
|
81
|
-
|
|
82
|
-
|
|
88
|
+
export function requiresCpuF16ToF32MatmulMaterialization(location, gpuCapabilities, keepF32Weights) {
|
|
89
|
+
if (location?.dtype !== 'F16' || !location?.role) return false;
|
|
90
|
+
return resolveMatmulWeightDtype(location, gpuCapabilities, keepF32Weights) === 'f32';
|
|
83
91
|
}
|
|
84
92
|
|
|
85
93
|
export function resolveWeightLayout(location) {
|
|
@@ -39,6 +39,8 @@ export class ShardCache {
|
|
|
39
39
|
get hasCustomRangeLoader(): boolean;
|
|
40
40
|
get hasCustomStreamLoader(): boolean;
|
|
41
41
|
get canStreamRanges(): boolean;
|
|
42
|
+
resetCustomReadStats(): void;
|
|
43
|
+
get customReadStats(): { bytesRead: number; shardsRead: number };
|
|
42
44
|
has(shardIndex: number): boolean;
|
|
43
45
|
get size(): number;
|
|
44
46
|
get totalBytes(): number;
|
|
@@ -25,6 +25,8 @@ export class ShardCache {
|
|
|
25
25
|
#highPriorityQueue = [];
|
|
26
26
|
#lowPriorityQueue = [];
|
|
27
27
|
#epoch = 0;
|
|
28
|
+
#customReadBytes = 0;
|
|
29
|
+
#customReadShards = new Set();
|
|
28
30
|
|
|
29
31
|
lastSource = null;
|
|
30
32
|
|
|
@@ -114,6 +116,18 @@ export class ShardCache {
|
|
|
114
116
|
}
|
|
115
117
|
}
|
|
116
118
|
|
|
119
|
+
resetCustomReadStats() {
|
|
120
|
+
this.#customReadBytes = 0;
|
|
121
|
+
this.#customReadShards.clear();
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
get customReadStats() {
|
|
125
|
+
return {
|
|
126
|
+
bytesRead: this.#customReadBytes,
|
|
127
|
+
shardsRead: this.#customReadShards.size,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
117
131
|
setManifest(manifest) {
|
|
118
132
|
this.#manifest = manifest;
|
|
119
133
|
}
|
|
@@ -228,6 +242,13 @@ export class ShardCache {
|
|
|
228
242
|
throw new Error('Custom shard loader must return ArrayBuffer or Uint8Array.');
|
|
229
243
|
}
|
|
230
244
|
|
|
245
|
+
#recordCustomRead(shardIndex, byteLength) {
|
|
246
|
+
const bytes = Number(byteLength);
|
|
247
|
+
if (!Number.isFinite(bytes) || bytes <= 0) return;
|
|
248
|
+
this.#customReadBytes += Math.floor(bytes);
|
|
249
|
+
this.#customReadShards.add(shardIndex);
|
|
250
|
+
}
|
|
251
|
+
|
|
231
252
|
#throwShortStreamRead(shardIndex, start, want, produced, path) {
|
|
232
253
|
throw new Error(
|
|
233
254
|
`Shard ${shardIndex} short stream read via ${path}: ` +
|
|
@@ -257,7 +278,9 @@ export class ShardCache {
|
|
|
257
278
|
const rangeData = await this.#customRangeLoader(shardIndex, start, want);
|
|
258
279
|
const elapsed = (performance.now() - rangeStart) / 1000;
|
|
259
280
|
this.#setLastSource('custom', elapsed, 'range', 'custom-range');
|
|
260
|
-
|
|
281
|
+
const buffer = this.#toArrayBuffer(rangeData);
|
|
282
|
+
this.#recordCustomRead(shardIndex, buffer.byteLength);
|
|
283
|
+
return buffer;
|
|
261
284
|
} catch (error) {
|
|
262
285
|
const unsupported = this.#isUnsupportedRangeOrStream(error);
|
|
263
286
|
if (!unsupported) {
|
|
@@ -337,6 +360,7 @@ export class ShardCache {
|
|
|
337
360
|
const bytes = chunk instanceof Uint8Array ? chunk : new Uint8Array(this.#toArrayBuffer(chunk));
|
|
338
361
|
if (bytes.byteLength > 0) {
|
|
339
362
|
produced += bytes.byteLength;
|
|
363
|
+
this.#recordCustomRead(shardIndex, bytes.byteLength);
|
|
340
364
|
yield bytes;
|
|
341
365
|
}
|
|
342
366
|
}
|
|
@@ -367,6 +391,7 @@ export class ShardCache {
|
|
|
367
391
|
}
|
|
368
392
|
emptyRetries = 0;
|
|
369
393
|
resumed += bytes.byteLength;
|
|
394
|
+
this.#recordCustomRead(shardIndex, bytes.byteLength);
|
|
370
395
|
yield bytes;
|
|
371
396
|
}
|
|
372
397
|
if (want != null && produced + resumed < want) {
|
|
@@ -411,6 +436,7 @@ export class ShardCache {
|
|
|
411
436
|
}
|
|
412
437
|
emptyRetries = 0;
|
|
413
438
|
resumed += bytes.byteLength;
|
|
439
|
+
this.#recordCustomRead(shardIndex, bytes.byteLength);
|
|
414
440
|
yield bytes;
|
|
415
441
|
}
|
|
416
442
|
if (produced + resumed < want) {
|
|
@@ -462,6 +488,7 @@ export class ShardCache {
|
|
|
462
488
|
}
|
|
463
489
|
emptyRetries = 0;
|
|
464
490
|
produced += bytes.byteLength;
|
|
491
|
+
this.#recordCustomRead(shardIndex, bytes.byteLength);
|
|
465
492
|
yield bytes;
|
|
466
493
|
if (bytes.byteLength < requestLength) {
|
|
467
494
|
partialRetryUsed = true;
|
|
@@ -528,6 +555,7 @@ export class ShardCache {
|
|
|
528
555
|
|
|
529
556
|
// Normalize to ArrayBuffer for downstream slicing
|
|
530
557
|
const arrayBuffer = this.#toArrayBuffer(data);
|
|
558
|
+
this.#recordCustomRead(shardIndex, arrayBuffer.byteLength);
|
|
531
559
|
|
|
532
560
|
if (epoch === this.#epoch) {
|
|
533
561
|
this.#add(shardIndex, arrayBuffer);
|
package/src/models/gemma4.js
CHANGED
|
@@ -23,6 +23,14 @@ const family = createFamily({
|
|
|
23
23
|
defaultRuntimeProfile: 'profiles/throughput',
|
|
24
24
|
modes: ['text', 'vision'],
|
|
25
25
|
},
|
|
26
|
+
{
|
|
27
|
+
modelId: 'gemma-4-e2b-it-q4k-ehf16-af16-int4ple',
|
|
28
|
+
label: 'Gemma 4 E2B Instruct (Q4K/F16a/INT4 PLE)',
|
|
29
|
+
sourceModel: 'google/gemma-4-e2b-it',
|
|
30
|
+
hfPath: 'models/gemma-4-e2b-it-q4k-ehf16-af16-int4ple',
|
|
31
|
+
defaultRuntimeProfile: 'profiles/throughput',
|
|
32
|
+
modes: ['text', 'vision'],
|
|
33
|
+
},
|
|
26
34
|
{
|
|
27
35
|
modelId: 'gemma-4-12b-it-text-w4a16-ct-ehf16-af16',
|
|
28
36
|
label: 'Gemma 4 12B Instruct Text (W4A16 QAT/F16a)',
|