doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -14,6 +14,11 @@ const QUANT_TAG_ALIASES = {
|
|
|
14
14
|
'q4k': 'q4k',
|
|
15
15
|
'q4': 'q4k',
|
|
16
16
|
'q4km': 'q4k',
|
|
17
|
+
// Q4_0 variants
|
|
18
|
+
'q4_0': 'q4_0',
|
|
19
|
+
'q4-0': 'q4_0',
|
|
20
|
+
'gguf-q4_0': 'q4_0',
|
|
21
|
+
'gguf_q4_0': 'q4_0',
|
|
17
22
|
// Q6_K variants
|
|
18
23
|
'q6_k': 'q6k',
|
|
19
24
|
'q6k': 'q6k',
|
|
@@ -21,6 +26,15 @@ const QUANT_TAG_ALIASES = {
|
|
|
21
26
|
// Q8_0 variants
|
|
22
27
|
'q8_0': 'q8_0',
|
|
23
28
|
'q8': 'q8_0',
|
|
29
|
+
// Compressed-tensors / mobile QAT variants
|
|
30
|
+
'w4a16': 'w4a16',
|
|
31
|
+
'w4a16-ct': 'w4a16',
|
|
32
|
+
'w4a16_ct': 'w4a16',
|
|
33
|
+
'compressed-tensors-w4a16': 'w4a16',
|
|
34
|
+
'compressed_tensors_w4a16': 'w4a16',
|
|
35
|
+
'wna8o8': 'wna8o8',
|
|
36
|
+
'wna8-o8': 'wna8o8',
|
|
37
|
+
'wna8_o8': 'wna8o8',
|
|
24
38
|
// MXFP4 variants
|
|
25
39
|
'mxfp4': 'mxfp4',
|
|
26
40
|
'mxp4': 'mxfp4',
|
|
@@ -62,9 +76,47 @@ const PER_LAYER_EMBEDDING_VARIANT_TAGS = {
|
|
|
62
76
|
int4_per_row: 'int4ple',
|
|
63
77
|
};
|
|
64
78
|
|
|
79
|
+
const SOURCE_TRAINING_QUANTIZATION_ALIASES = {
|
|
80
|
+
'qat': 'qat',
|
|
81
|
+
'quantization-aware-training': 'qat',
|
|
82
|
+
'quantization_aware_training': 'qat',
|
|
83
|
+
'ptq': 'ptq',
|
|
84
|
+
'post-training-quantization': 'ptq',
|
|
85
|
+
'post_training_quantization': 'ptq',
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const SOURCE_QUANTIZATION_TARGET_ALIASES = {
|
|
89
|
+
'q4_0': 'q4_0',
|
|
90
|
+
'q4-0': 'q4_0',
|
|
91
|
+
'gguf-q4_0': 'q4_0',
|
|
92
|
+
'gguf_q4_0': 'q4_0',
|
|
93
|
+
'w4a16': 'w4a16',
|
|
94
|
+
'w4a16-ct': 'w4a16',
|
|
95
|
+
'w4a16_ct': 'w4a16',
|
|
96
|
+
'compressed-tensors-w4a16': 'w4a16',
|
|
97
|
+
'compressed_tensors_w4a16': 'w4a16',
|
|
98
|
+
'wna8o8': 'wNa8o8',
|
|
99
|
+
'wna8-o8': 'wNa8o8',
|
|
100
|
+
'wna8_o8': 'wNa8o8',
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
const SOURCE_QUANTIZATION_TARGET_TAGS = {
|
|
104
|
+
q4_0: 'q4_0',
|
|
105
|
+
w4a16: 'w4a16',
|
|
106
|
+
wNa8o8: 'wna8o8',
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
const SOURCE_QUANTIZATION_FORMAT_ALIASES = {
|
|
110
|
+
'compressed-tensors': 'compressed-tensors',
|
|
111
|
+
'compressed_tensors': 'compressed-tensors',
|
|
112
|
+
'compressedtensors': 'compressed-tensors',
|
|
113
|
+
'ct': 'compressed-tensors',
|
|
114
|
+
'gguf': 'gguf',
|
|
115
|
+
};
|
|
116
|
+
|
|
65
117
|
export function normalizeQuantTag(value) {
|
|
66
118
|
if (!value) return DEFAULT_QUANT_TAG;
|
|
67
|
-
const lower = value.toLowerCase();
|
|
119
|
+
const lower = String(value).trim().toLowerCase();
|
|
68
120
|
return QUANT_TAG_ALIASES[lower] ?? lower;
|
|
69
121
|
}
|
|
70
122
|
|
|
@@ -72,7 +124,7 @@ function validateQuantType(value, source) {
|
|
|
72
124
|
if (!value) return;
|
|
73
125
|
const normalized = normalizeQuantTag(value);
|
|
74
126
|
|
|
75
|
-
const supported = ['q4k', 'f16', 'bf16', 'f32'];
|
|
127
|
+
const supported = ['q4k', 'q4_0', 'w4a16', 'wna8o8', 'f16', 'bf16', 'f32'];
|
|
76
128
|
if (supported.includes(normalized)) return;
|
|
77
129
|
|
|
78
130
|
const planned = ['q6k', 'q8_0', 'fp8e4', 'fp8e5', 'i4', 'i8'];
|
|
@@ -103,12 +155,110 @@ function perLayerEmbeddingVariantTag(value) {
|
|
|
103
155
|
return PER_LAYER_EMBEDDING_VARIANT_TAGS[value] ?? null;
|
|
104
156
|
}
|
|
105
157
|
|
|
158
|
+
function normalizeSourceTrainingQuantization(value) {
|
|
159
|
+
if (value == null) return null;
|
|
160
|
+
const normalized = String(value).trim().toLowerCase().replace(/\s+/g, '-');
|
|
161
|
+
if (!normalized) return null;
|
|
162
|
+
const canonical = SOURCE_TRAINING_QUANTIZATION_ALIASES[normalized];
|
|
163
|
+
if (canonical) return canonical;
|
|
164
|
+
throw new Error(
|
|
165
|
+
`converter.quantization.sourceTrainingQuantization must be "qat", "ptq", or null; got ${JSON.stringify(value)}.`
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function normalizeSourceQuantizationTarget(value) {
|
|
170
|
+
if (value == null) return null;
|
|
171
|
+
const normalized = String(value).trim().toLowerCase().replace(/\s+/g, '-');
|
|
172
|
+
if (!normalized) return null;
|
|
173
|
+
const canonical = SOURCE_QUANTIZATION_TARGET_ALIASES[normalized];
|
|
174
|
+
if (canonical) return canonical;
|
|
175
|
+
throw new Error(
|
|
176
|
+
`converter.quantization.sourceQuantizationTarget must be "q4_0", "w4a16", "wNa8o8", or null; got ${JSON.stringify(value)}.`
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function normalizeSourceQuantizationFormat(value) {
|
|
181
|
+
if (value == null) return null;
|
|
182
|
+
const normalized = String(value).trim().toLowerCase().replace(/\s+/g, '-');
|
|
183
|
+
if (!normalized) return null;
|
|
184
|
+
const canonical = SOURCE_QUANTIZATION_FORMAT_ALIASES[normalized];
|
|
185
|
+
if (canonical) return canonical;
|
|
186
|
+
throw new Error(
|
|
187
|
+
`converter.quantization.sourceQuantizationFormat must be "compressed-tensors", "gguf", or null; got ${JSON.stringify(value)}.`
|
|
188
|
+
);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function sourceTargetQuantTag(sourceQuantizationTarget) {
|
|
192
|
+
if (!sourceQuantizationTarget) return null;
|
|
193
|
+
return SOURCE_QUANTIZATION_TARGET_TAGS[sourceQuantizationTarget]
|
|
194
|
+
?? normalizeQuantTag(sourceQuantizationTarget);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function applySourceQuantizationMetadata(info, sourceTrainingQuantization, sourceQuantizationTarget, sourceQuantizationFormat) {
|
|
198
|
+
if (sourceQuantizationTarget && !sourceTrainingQuantization) {
|
|
199
|
+
throw new Error(
|
|
200
|
+
'converter.quantization.sourceQuantizationTarget requires ' +
|
|
201
|
+
'converter.quantization.sourceTrainingQuantization.'
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
if (sourceTrainingQuantization) {
|
|
205
|
+
info.sourceTrainingQuantization = sourceTrainingQuantization;
|
|
206
|
+
} else {
|
|
207
|
+
delete info.sourceTrainingQuantization;
|
|
208
|
+
}
|
|
209
|
+
if (sourceQuantizationTarget) {
|
|
210
|
+
info.sourceQuantizationTarget = sourceQuantizationTarget;
|
|
211
|
+
} else {
|
|
212
|
+
delete info.sourceQuantizationTarget;
|
|
213
|
+
}
|
|
214
|
+
if (sourceQuantizationFormat) {
|
|
215
|
+
info.sourceQuantizationFormat = sourceQuantizationFormat;
|
|
216
|
+
} else {
|
|
217
|
+
delete info.sourceQuantizationFormat;
|
|
218
|
+
}
|
|
219
|
+
if (sourceTrainingQuantization !== 'qat' || !sourceQuantizationTarget) {
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const targetQuant = sourceTargetQuantTag(sourceQuantizationTarget);
|
|
224
|
+
const weightQuant = normalizeQuantTag(info.weights);
|
|
225
|
+
if (weightQuant !== targetQuant) {
|
|
226
|
+
throw new Error(
|
|
227
|
+
`QAT sourceQuantizationTarget="${sourceQuantizationTarget}" requires ` +
|
|
228
|
+
`quantizationInfo.weights="${targetQuant}"; got "${weightQuant}".`
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const lmHeadQuant = normalizeQuantTag(info.lmHead ?? info.embeddings ?? info.weights);
|
|
233
|
+
const embeddingQuant = normalizeQuantTag(info.embeddings ?? info.weights);
|
|
234
|
+
const allowsDenseTiedW4A16Head = (
|
|
235
|
+
sourceQuantizationTarget === 'w4a16'
|
|
236
|
+
&& sourceQuantizationFormat === 'compressed-tensors'
|
|
237
|
+
&& lmHeadQuant === 'f16'
|
|
238
|
+
&& embeddingQuant === 'f16'
|
|
239
|
+
);
|
|
240
|
+
if (lmHeadQuant !== targetQuant) {
|
|
241
|
+
if (allowsDenseTiedW4A16Head) {
|
|
242
|
+
info.lmHead = 'f16';
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
throw new Error(
|
|
246
|
+
`QAT sourceQuantizationTarget="${sourceQuantizationTarget}" requires ` +
|
|
247
|
+
`quantizationInfo.lmHead="${targetQuant}" so the LM head stays quantized; got "${lmHeadQuant}".`
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
info.lmHead = targetQuant;
|
|
251
|
+
}
|
|
252
|
+
|
|
106
253
|
|
|
107
254
|
// Canonical dtype to manifest format mapping.
|
|
108
255
|
const MANIFEST_QUANT_NAMES = {
|
|
109
256
|
'q4k': 'Q4_K_M',
|
|
257
|
+
'q4_0': 'Q4_0',
|
|
110
258
|
'q6k': 'Q6_K',
|
|
111
259
|
'q8_0': 'Q8_0',
|
|
260
|
+
'w4a16': 'W4A16',
|
|
261
|
+
'wna8o8': 'WNA8O8',
|
|
112
262
|
};
|
|
113
263
|
|
|
114
264
|
export function resolveManifestQuantization(quantize, fallback) {
|
|
@@ -262,6 +412,15 @@ export function buildQuantizationInfo(
|
|
|
262
412
|
const projectorQuant = quantization.projector ?? null;
|
|
263
413
|
const computePrecision = quantization.computePrecision ?? null;
|
|
264
414
|
const perLayerEmbeddings = normalizePerLayerEmbeddingQuant(quantization.perLayerEmbeddings ?? null);
|
|
415
|
+
const sourceTrainingQuantization = normalizeSourceTrainingQuantization(
|
|
416
|
+
quantization.sourceTrainingQuantization ?? null
|
|
417
|
+
);
|
|
418
|
+
const sourceQuantizationTarget = normalizeSourceQuantizationTarget(
|
|
419
|
+
quantization.sourceQuantizationTarget ?? null
|
|
420
|
+
);
|
|
421
|
+
const sourceQuantizationFormat = normalizeSourceQuantizationFormat(
|
|
422
|
+
quantization.sourceQuantizationFormat ?? null
|
|
423
|
+
);
|
|
265
424
|
|
|
266
425
|
validateQuantType(weightQuant, 'converter.quantization.weights');
|
|
267
426
|
validateQuantType(embedQuant, 'converter.quantization.embeddings');
|
|
@@ -342,6 +501,12 @@ export function buildQuantizationInfo(
|
|
|
342
501
|
if (perLayerEmbeddings) {
|
|
343
502
|
info.perLayerEmbeddings = perLayerEmbeddings;
|
|
344
503
|
}
|
|
504
|
+
applySourceQuantizationMetadata(
|
|
505
|
+
info,
|
|
506
|
+
sourceTrainingQuantization,
|
|
507
|
+
sourceQuantizationTarget,
|
|
508
|
+
sourceQuantizationFormat
|
|
509
|
+
);
|
|
345
510
|
|
|
346
511
|
// Q4K layout: 'row' (fused kernel compatible) or 'col' (dequant fallback)
|
|
347
512
|
// Default to 'row' for Q4K weights since that's the performant path
|
|
@@ -369,6 +534,9 @@ export function resolveModelId(modelId, baseName, variantTag) {
|
|
|
369
534
|
// Canonical dtype to WebGPU dtype mapping.
|
|
370
535
|
const WEBGPU_DTYPE_NAMES = {
|
|
371
536
|
'q4k': 'Q4_K_M',
|
|
537
|
+
'q4_0': 'Q4_0',
|
|
538
|
+
'w4a16': 'W4A16',
|
|
539
|
+
'wna8o8': 'WNA8O8',
|
|
372
540
|
'bf16': 'F16',
|
|
373
541
|
};
|
|
374
542
|
|
|
@@ -381,8 +549,11 @@ function normalizeStoredDtype(value) {
|
|
|
381
549
|
const upper = String(value).trim().toUpperCase();
|
|
382
550
|
if (!upper) return null;
|
|
383
551
|
if (upper === 'Q4_K' || upper === 'Q4_K_M') return 'q4k';
|
|
552
|
+
if (upper === 'Q4_0') return 'q4_0';
|
|
384
553
|
if (upper === 'Q6_K') return 'q6k';
|
|
385
554
|
if (upper === 'Q8_0') return 'q8_0';
|
|
555
|
+
if (upper === 'W4A16') return 'w4a16';
|
|
556
|
+
if (upper === 'WNA8O8') return 'wna8o8';
|
|
386
557
|
if (upper === 'BF16') return 'bf16';
|
|
387
558
|
if (upper === 'F16') return 'f16';
|
|
388
559
|
if (upper === 'F32') return 'f32';
|
|
@@ -453,6 +624,12 @@ export function resolveEffectiveQuantizationInfo(baseInfo, tensors) {
|
|
|
453
624
|
delete resolved.layout;
|
|
454
625
|
}
|
|
455
626
|
|
|
627
|
+
applySourceQuantizationMetadata(
|
|
628
|
+
resolved,
|
|
629
|
+
normalizeSourceTrainingQuantization(base.sourceTrainingQuantization ?? null),
|
|
630
|
+
normalizeSourceQuantizationTarget(base.sourceQuantizationTarget ?? null),
|
|
631
|
+
normalizeSourceQuantizationFormat(base.sourceQuantizationFormat ?? null)
|
|
632
|
+
);
|
|
456
633
|
resolved.variantTag = buildVariantTag(resolved);
|
|
457
634
|
return resolved;
|
|
458
635
|
}
|
|
@@ -41,10 +41,11 @@ export declare function quantizeQ4KBlock(data: Float32Array, offset: number): Ui
|
|
|
41
41
|
export declare function quantizeToQ4KM(data: Float32Array, shape: number[]): QuantizeResult;
|
|
42
42
|
|
|
43
43
|
/**
|
|
44
|
-
* Row-wise Q4K quantization for
|
|
45
|
-
*
|
|
44
|
+
* Row-wise Q4K quantization for matrix-like tensors.
|
|
45
|
+
* Leading dimensions are treated as batches of rows; each row is padded to
|
|
46
|
+
* the 256-element boundary before quantization.
|
|
46
47
|
*/
|
|
47
|
-
export declare function quantizeToQ4KMRowWise(data: Float32Array, shape: [
|
|
48
|
+
export declare function quantizeToQ4KMRowWise(data: Float32Array, shape: number[]): QuantizeResult;
|
|
48
49
|
|
|
49
50
|
/**
|
|
50
51
|
* Transpose a 2D F32 matrix.
|
|
@@ -75,7 +76,7 @@ export declare function dequantizeQ4KM(
|
|
|
75
76
|
|
|
76
77
|
export declare function dequantizeQ4KMRowWise(
|
|
77
78
|
quantized: Uint8Array,
|
|
78
|
-
shape: [
|
|
79
|
+
shape: number[]
|
|
79
80
|
): Float32Array;
|
|
80
81
|
|
|
81
82
|
export declare function calculateQuantizationError(
|
|
@@ -251,7 +251,11 @@ export function quantizeToQ4KM(data, shape) {
|
|
|
251
251
|
|
|
252
252
|
|
|
253
253
|
export function quantizeToQ4KMRowWise(data, shape) {
|
|
254
|
-
|
|
254
|
+
if (!Array.isArray(shape) || shape.length < 2) {
|
|
255
|
+
throw new Error(`Row-wise Q4K quantization requires a matrix-like shape, got ${shape}`);
|
|
256
|
+
}
|
|
257
|
+
const cols = shape[shape.length - 1];
|
|
258
|
+
const rows = shape.slice(0, -1).reduce((a, b) => a * b, 1);
|
|
255
259
|
const numElements = rows * cols;
|
|
256
260
|
|
|
257
261
|
if (data.length !== numElements) {
|
|
@@ -393,6 +397,12 @@ export function getQ4KSize(shape, layout = 'flat') {
|
|
|
393
397
|
const numElements = shape.reduce((a, b) => a * b, 1);
|
|
394
398
|
|
|
395
399
|
if (layout === 'flat' || shape.length !== 2) {
|
|
400
|
+
if (layout === 'row' && shape.length >= 2) {
|
|
401
|
+
const cols = shape[shape.length - 1];
|
|
402
|
+
const rows = shape.slice(0, -1).reduce((a, b) => a * b, 1);
|
|
403
|
+
const blocksPerRow = Math.ceil(cols / QK_K);
|
|
404
|
+
return rows * blocksPerRow * QK4_K_BLOCK_SIZE;
|
|
405
|
+
}
|
|
396
406
|
const numBlocks = Math.ceil(numElements / QK_K);
|
|
397
407
|
return numBlocks * QK4_K_BLOCK_SIZE;
|
|
398
408
|
}
|
|
@@ -433,7 +443,11 @@ export function dequantizeQ4KM(quantized, numBlocks, shape) {
|
|
|
433
443
|
}
|
|
434
444
|
|
|
435
445
|
export function dequantizeQ4KMRowWise(quantized, shape) {
|
|
436
|
-
|
|
446
|
+
if (!Array.isArray(shape) || shape.length < 2) {
|
|
447
|
+
throw new Error(`Row-wise Q4K dequantization requires a matrix-like shape, got ${shape}`);
|
|
448
|
+
}
|
|
449
|
+
const cols = shape[shape.length - 1];
|
|
450
|
+
const rows = shape.slice(0, -1).reduce((a, b) => a * b, 1);
|
|
437
451
|
const blocksPerRow = Math.ceil(cols / QK_K);
|
|
438
452
|
const result = new Float32Array(rows * cols);
|
|
439
453
|
|
|
@@ -10,11 +10,8 @@ import { log } from '../../debug/index.js';
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
const parseTensorName = (name) => {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
// - layers.12.gate_proj.lora_b
|
|
16
|
-
// - layer0.v_proj.lora_a
|
|
17
|
-
const match = name.match(/layers?\.?(\d+)\.([^.]+)\.lora_([ab])/i);
|
|
13
|
+
const text = String(name || '');
|
|
14
|
+
const match = text.match(/(?:^|\.)layers?\.?(\d+)\.(?:[A-Za-z0-9_]+\.)*([A-Za-z0-9_]+)\.lora_([ab])(?:\.[A-Za-z0-9_]+)?$/i);
|
|
18
15
|
if (!match) return null;
|
|
19
16
|
const layer = parseInt(match[1], 10);
|
|
20
17
|
const rawModule = match[2].toLowerCase();
|
|
@@ -76,12 +73,81 @@ const validateShape = (tensor, data) => {
|
|
|
76
73
|
};
|
|
77
74
|
|
|
78
75
|
|
|
76
|
+
function asArrayBuffer(data) {
|
|
77
|
+
if (data instanceof ArrayBuffer) return data;
|
|
78
|
+
if (ArrayBuffer.isView(data)) {
|
|
79
|
+
return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
|
80
|
+
}
|
|
81
|
+
throw new Error('Expected ArrayBuffer or typed array data');
|
|
82
|
+
}
|
|
83
|
+
|
|
79
84
|
async function computeSHA256(data) {
|
|
80
|
-
const hashBuffer = await crypto.subtle.digest('SHA-256', data);
|
|
85
|
+
const hashBuffer = await crypto.subtle.digest('SHA-256', asArrayBuffer(data));
|
|
81
86
|
const hashArray = new Uint8Array(hashBuffer);
|
|
82
87
|
return Array.from(hashArray).map(b => b.toString(16).padStart(2, '0')).join('');
|
|
83
88
|
}
|
|
84
89
|
|
|
90
|
+
function stripHashPrefix(value) {
|
|
91
|
+
return String(value || '').replace(/^sha256:/i, '').toLowerCase();
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function isUrl(value) {
|
|
95
|
+
return /^https?:\/\//i.test(String(value || ''));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function readPathBytes(sourcePath, options = {}) {
|
|
99
|
+
const text = String(sourcePath || '').trim();
|
|
100
|
+
if (!text) {
|
|
101
|
+
throw new Error('LoRA weightsPath is required');
|
|
102
|
+
}
|
|
103
|
+
if (isUrl(text)) {
|
|
104
|
+
if (options.fetchUrl) return options.fetchUrl(text);
|
|
105
|
+
const res = await fetch(text);
|
|
106
|
+
if (!res.ok) {
|
|
107
|
+
throw new Error(`Failed to fetch LoRA weights: ${res.status}`);
|
|
108
|
+
}
|
|
109
|
+
return res.arrayBuffer();
|
|
110
|
+
}
|
|
111
|
+
if (options.readFile) {
|
|
112
|
+
const path = options.resolvePath
|
|
113
|
+
? options.resolvePath(text)
|
|
114
|
+
: (options.basePath ? `${String(options.basePath).replace(/\/$/, '')}/${text}` : text);
|
|
115
|
+
return options.readFile(path);
|
|
116
|
+
}
|
|
117
|
+
if (options.readOPFS) {
|
|
118
|
+
return options.readOPFS(text);
|
|
119
|
+
}
|
|
120
|
+
if (options.fetchUrl) {
|
|
121
|
+
return options.fetchUrl(text);
|
|
122
|
+
}
|
|
123
|
+
throw new Error('Cannot load LoRA weightsPath without fetchUrl, readFile, or readOPFS');
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
async function assertChecksum(data, manifest) {
|
|
127
|
+
const expected = stripHashPrefix(manifest.checksum);
|
|
128
|
+
if (!expected) return;
|
|
129
|
+
const algorithm = String(manifest.checksumAlgorithm || 'sha256').toLowerCase();
|
|
130
|
+
if (algorithm !== 'sha256') {
|
|
131
|
+
throw new Error(`Unsupported LoRA checksum algorithm: ${algorithm}`);
|
|
132
|
+
}
|
|
133
|
+
const computed = await computeSHA256(data);
|
|
134
|
+
if (computed.toLowerCase() !== expected) {
|
|
135
|
+
throw new Error(`LoRA checksum mismatch: expected ${expected}, got ${computed}`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function loadExternalWeights(manifest, options = {}) {
|
|
140
|
+
const format = String(manifest.weightsFormat || 'safetensors').toLowerCase();
|
|
141
|
+
if (format !== 'safetensors') {
|
|
142
|
+
throw new Error(`Unsupported LoRA weightsFormat for external weights: ${format}`);
|
|
143
|
+
}
|
|
144
|
+
const data = await readPathBytes(manifest.weightsPath, options);
|
|
145
|
+
if (!options.skipVerify) {
|
|
146
|
+
await assertChecksum(data, manifest);
|
|
147
|
+
}
|
|
148
|
+
return loadLoRAFromSafetensors(asArrayBuffer(data), manifest);
|
|
149
|
+
}
|
|
150
|
+
|
|
85
151
|
function assertCompleteAdapterLayers(adapter) {
|
|
86
152
|
for (const [layerIndex, layer] of adapter.layers.entries()) {
|
|
87
153
|
for (const [moduleName, weights] of Object.entries(layer)) {
|
|
@@ -102,6 +168,22 @@ function assertCompleteAdapterLayers(adapter) {
|
|
|
102
168
|
|
|
103
169
|
|
|
104
170
|
export async function loadLoRAWeights(path, options = {}) {
|
|
171
|
+
if (path && typeof path === 'object') {
|
|
172
|
+
const manifest = applyAdapterManifestDefaults(path);
|
|
173
|
+
const validation = validateManifest(manifest);
|
|
174
|
+
if (!validation.valid) {
|
|
175
|
+
const errors = validation.errors.map(e => `${e.field}: ${e.message}`).join('; ');
|
|
176
|
+
throw new Error(`Invalid LoRA manifest: ${errors}`);
|
|
177
|
+
}
|
|
178
|
+
const adapter = await loadLoRAFromManifest(manifest, options);
|
|
179
|
+
return {
|
|
180
|
+
adapter,
|
|
181
|
+
manifest,
|
|
182
|
+
loadedFromCache: false,
|
|
183
|
+
checksumValid: manifest.checksum && !options.skipVerify ? true : undefined,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
|
|
105
187
|
let manifestJson;
|
|
106
188
|
let loadedFromCache = false;
|
|
107
189
|
|
|
@@ -159,26 +241,16 @@ export async function loadLoRAWeights(path, options = {}) {
|
|
|
159
241
|
return res.arrayBuffer();
|
|
160
242
|
};
|
|
161
243
|
|
|
162
|
-
// Load adapter
|
|
163
244
|
const adapter = await loadLoRAFromManifest(
|
|
164
245
|
manifest,
|
|
165
246
|
{ ...options, fetchUrl: fetchWithBase }
|
|
166
247
|
);
|
|
167
248
|
|
|
168
|
-
// Verify checksum if provided
|
|
169
249
|
let checksumValid;
|
|
170
|
-
if (manifest.checksum && !options.skipVerify) {
|
|
250
|
+
if (manifest.checksum && !options.skipVerify && Array.isArray(manifest.tensors) && manifest.tensors.length > 0) {
|
|
171
251
|
const algorithm = manifest.checksumAlgorithm;
|
|
172
252
|
if (algorithm !== 'sha256') {
|
|
173
253
|
throw new Error(`Unsupported LoRA checksum algorithm: ${algorithm}`);
|
|
174
|
-
} else if (manifest.weightsPath) {
|
|
175
|
-
// Compute checksum of the weights file
|
|
176
|
-
const weightsData = await fetchWithBase(manifest.weightsPath);
|
|
177
|
-
const computedHash = await computeSHA256(weightsData);
|
|
178
|
-
checksumValid = computedHash.toLowerCase() === manifest.checksum.toLowerCase();
|
|
179
|
-
if (!checksumValid) {
|
|
180
|
-
throw new Error(`LoRA checksum mismatch: expected ${manifest.checksum}, got ${computedHash}`);
|
|
181
|
-
}
|
|
182
254
|
} else if (manifest.tensors && manifest.tensors.length > 0) {
|
|
183
255
|
// For inline tensors, compute checksum over concatenated tensor data
|
|
184
256
|
const tensorBuffers = [];
|
|
@@ -199,7 +271,7 @@ export async function loadLoRAWeights(path, options = {}) {
|
|
|
199
271
|
offset += buf.byteLength;
|
|
200
272
|
}
|
|
201
273
|
const computedHash = await computeSHA256(combined.buffer);
|
|
202
|
-
checksumValid = computedHash.toLowerCase() === manifest.checksum
|
|
274
|
+
checksumValid = computedHash.toLowerCase() === stripHashPrefix(manifest.checksum);
|
|
203
275
|
if (!checksumValid) {
|
|
204
276
|
throw new Error(`LoRA checksum mismatch: expected ${manifest.checksum}, got ${computedHash}`);
|
|
205
277
|
}
|
|
@@ -211,12 +283,16 @@ export async function loadLoRAWeights(path, options = {}) {
|
|
|
211
283
|
adapter,
|
|
212
284
|
manifest,
|
|
213
285
|
loadedFromCache,
|
|
214
|
-
checksumValid,
|
|
286
|
+
checksumValid: manifest.checksum && !options.skipVerify && checksumValid !== false ? true : checksumValid,
|
|
215
287
|
};
|
|
216
288
|
}
|
|
217
289
|
|
|
218
290
|
|
|
219
291
|
export async function loadLoRAFromManifest(manifest, options = {}) {
|
|
292
|
+
if (manifest?.weightsPath && !Array.isArray(manifest.tensors)) {
|
|
293
|
+
return loadExternalWeights(manifest, options);
|
|
294
|
+
}
|
|
295
|
+
|
|
220
296
|
const adapter = {
|
|
221
297
|
name: manifest.name,
|
|
222
298
|
version: manifest.version,
|
|
@@ -313,11 +389,12 @@ export function applyDeltaWeights(baseWeight, loraA, loraB, scale) {
|
|
|
313
389
|
|
|
314
390
|
|
|
315
391
|
export async function loadLoRAFromSafetensors(data, manifest) {
|
|
392
|
+
const buffer = asArrayBuffer(data);
|
|
316
393
|
// Parse safetensors header
|
|
317
|
-
const view = new DataView(
|
|
394
|
+
const view = new DataView(buffer);
|
|
318
395
|
const headerSize = Number(view.getBigUint64(0, true));
|
|
319
396
|
const headerJson = new TextDecoder().decode(
|
|
320
|
-
new Uint8Array(
|
|
397
|
+
new Uint8Array(buffer, 8, headerSize)
|
|
321
398
|
);
|
|
322
399
|
const header = JSON.parse(headerJson);
|
|
323
400
|
|
|
@@ -342,15 +419,17 @@ export async function loadLoRAFromSafetensors(data, manifest) {
|
|
|
342
419
|
}
|
|
343
420
|
|
|
344
421
|
const [start, end] = tensorInfo.data_offsets;
|
|
345
|
-
const tensorData = new Uint8Array(
|
|
422
|
+
const tensorData = new Uint8Array(buffer, dataOffset + start, end - start);
|
|
346
423
|
|
|
347
424
|
// Convert to Float32Array based on dtype
|
|
348
425
|
let floatData;
|
|
349
426
|
if (tensorInfo.dtype === 'F32') {
|
|
350
|
-
|
|
427
|
+
const aligned = tensorData.slice();
|
|
428
|
+
floatData = new Float32Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 4);
|
|
351
429
|
} else if (tensorInfo.dtype === 'F16' || tensorInfo.dtype === 'BF16') {
|
|
352
430
|
// Convert from f16/bf16 to f32
|
|
353
|
-
const
|
|
431
|
+
const aligned = tensorData.slice();
|
|
432
|
+
const f16View = new Uint16Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 2);
|
|
354
433
|
floatData = new Float32Array(f16View.length);
|
|
355
434
|
for (let i = 0; i < f16View.length; i++) {
|
|
356
435
|
if (tensorInfo.dtype === 'F16') {
|
|
@@ -364,6 +443,12 @@ export async function loadLoRAFromSafetensors(data, manifest) {
|
|
|
364
443
|
continue;
|
|
365
444
|
}
|
|
366
445
|
|
|
446
|
+
const shape = Array.isArray(tensorInfo.shape) ? tensorInfo.shape : [];
|
|
447
|
+
const expected = shape.reduce((product, value) => product * Number(value || 0), 1);
|
|
448
|
+
if (shape.length !== 2 || floatData.length !== expected) {
|
|
449
|
+
throw new Error(`LoRA safetensors tensor ${tensorName} shape mismatch: expected ${expected}, got ${floatData.length}`);
|
|
450
|
+
}
|
|
451
|
+
|
|
367
452
|
const layer = adapter.layers.get(parsed.layer) || {};
|
|
368
453
|
const scale = manifest.rank > 0 ? manifest.alpha / manifest.rank : 1;
|
|
369
454
|
|
|
@@ -1,4 +1,28 @@
|
|
|
1
1
|
export { DiffusionPipeline, createDiffusionPipeline } from '../../inference/pipelines/diffusion/pipeline.js';
|
|
2
|
+
export { DiffusionGemmaPipeline } from '../../inference/pipelines/diffusion-gemma/pipeline.js';
|
|
3
|
+
export type {
|
|
4
|
+
DiffusionGemmaGenerateOptions,
|
|
5
|
+
DiffusionGemmaPipelineContexts,
|
|
6
|
+
DiffusionGemmaStats,
|
|
7
|
+
} from '../../inference/pipelines/diffusion-gemma/pipeline.js';
|
|
8
|
+
export {
|
|
9
|
+
applyEntropyBoundStep,
|
|
10
|
+
createSeededRandom,
|
|
11
|
+
denoiseCanvas,
|
|
12
|
+
initializeCanvas,
|
|
13
|
+
parseDiffusionGemmaConfig,
|
|
14
|
+
resolveDenoisingTemperature,
|
|
15
|
+
updateStabilityState,
|
|
16
|
+
} from '../../inference/pipelines/diffusion-gemma/index.js';
|
|
17
|
+
export type {
|
|
18
|
+
DiffusionGemmaConfig,
|
|
19
|
+
DiffusionGemmaDenoiseResult,
|
|
20
|
+
DiffusionGemmaLogitsProvider,
|
|
21
|
+
DiffusionGemmaLogitsRequest,
|
|
22
|
+
DiffusionGemmaRouterContract,
|
|
23
|
+
DiffusionGemmaStabilityState,
|
|
24
|
+
DiffusionGemmaStepResult,
|
|
25
|
+
} from '../../inference/pipelines/diffusion-gemma/index.js';
|
|
2
26
|
export { createDiffusionWeightLoader } from '../../inference/pipelines/diffusion/weights.js';
|
|
3
27
|
export { mergeDiffusionConfig, initializeDiffusion } from '../../inference/pipelines/diffusion/init.js';
|
|
4
28
|
export {
|
|
@@ -1,4 +1,14 @@
|
|
|
1
1
|
export { DiffusionPipeline, createDiffusionPipeline } from '../../inference/pipelines/diffusion/pipeline.js';
|
|
2
|
+
export { DiffusionGemmaPipeline } from '../../inference/pipelines/diffusion-gemma/pipeline.js';
|
|
3
|
+
export {
|
|
4
|
+
applyEntropyBoundStep,
|
|
5
|
+
createSeededRandom,
|
|
6
|
+
denoiseCanvas,
|
|
7
|
+
initializeCanvas,
|
|
8
|
+
parseDiffusionGemmaConfig,
|
|
9
|
+
resolveDenoisingTemperature,
|
|
10
|
+
updateStabilityState,
|
|
11
|
+
} from '../../inference/pipelines/diffusion-gemma/index.js';
|
|
2
12
|
export { createDiffusionWeightLoader } from '../../inference/pipelines/diffusion/weights.js';
|
|
3
13
|
export { mergeDiffusionConfig, initializeDiffusion } from '../../inference/pipelines/diffusion/init.js';
|
|
4
14
|
export {
|