doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
|
@@ -9,13 +9,14 @@
|
|
|
9
9
|
import type { CpuWeightBuffer, WeightBuffer } from '../gpu/weight-buffer.js';
|
|
10
10
|
import type { TensorRole } from '../config/schema/index.js';
|
|
11
11
|
import type { TensorSourceTransform } from '../formats/rdrr/index.js';
|
|
12
|
+
import type { FunctionalDescriptorManifest } from '../formats/rdrr/functional-descriptor.js';
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* Tensor location in loaded model
|
|
15
16
|
*/
|
|
16
17
|
export interface TensorLocation {
|
|
17
|
-
shardIndex
|
|
18
|
-
offset
|
|
18
|
+
shardIndex?: number;
|
|
19
|
+
offset?: number;
|
|
19
20
|
size: number;
|
|
20
21
|
shape: number[];
|
|
21
22
|
dtype: string;
|
|
@@ -28,6 +29,8 @@ export interface TensorLocation {
|
|
|
28
29
|
originalShape?: number[];
|
|
29
30
|
/** Optional direct-source transform applied before dtype-specific loading */
|
|
30
31
|
sourceTransform?: TensorSourceTransform;
|
|
32
|
+
/** Parsed manifoldgguf descriptor for FUNCTIONAL_DESCRIPTOR tensors */
|
|
33
|
+
descriptorManifest?: FunctionalDescriptorManifest;
|
|
31
34
|
}
|
|
32
35
|
|
|
33
36
|
/**
|
|
@@ -53,7 +56,10 @@ export interface LayerWeights {
|
|
|
53
56
|
kNorm: GPUBuffer | Float32Array | null;
|
|
54
57
|
postAttentionNorm: GPUBuffer | Float32Array | null;
|
|
55
58
|
preFeedforwardNorm: GPUBuffer | Float32Array | null;
|
|
59
|
+
preFeedforwardNorm2?: GPUBuffer | Float32Array | null;
|
|
56
60
|
postFeedforwardNorm: GPUBuffer | Float32Array | null;
|
|
61
|
+
postFeedforwardNorm1?: GPUBuffer | Float32Array | null;
|
|
62
|
+
postFeedforwardNorm2?: GPUBuffer | Float32Array | null;
|
|
57
63
|
postNorm: GPUBuffer | Float32Array | null;
|
|
58
64
|
postAttnNorm: GPUBuffer | Float32Array | null;
|
|
59
65
|
convInProj?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
@@ -72,6 +78,8 @@ export interface LayerWeights {
|
|
|
72
78
|
gateUp?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
73
79
|
routerWeight?: GPUBuffer | import('../gpu/weight-buffer.js').WeightBuffer | Float32Array | null;
|
|
74
80
|
routerBias?: GPUBuffer | Float32Array | null;
|
|
81
|
+
routerScale?: GPUBuffer | Float32Array | null;
|
|
82
|
+
routerPerExpertScale?: GPUBuffer | Float32Array | null;
|
|
75
83
|
attentionSinks?: GPUBuffer | Float32Array | null;
|
|
76
84
|
perLayerInputGate?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
77
85
|
perLayerProjection?: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
@@ -79,6 +87,14 @@ export interface LayerWeights {
|
|
|
79
87
|
layerScalar?: GPUBuffer | Float32Array | null;
|
|
80
88
|
}
|
|
81
89
|
|
|
90
|
+
export interface DiffusionGemmaSelfConditioningWeights {
|
|
91
|
+
preNorm: GPUBuffer | Float32Array;
|
|
92
|
+
postNorm: GPUBuffer | Float32Array | null;
|
|
93
|
+
gateProj: GPUBuffer | WeightBuffer | Float32Array;
|
|
94
|
+
upProj: GPUBuffer | WeightBuffer | Float32Array;
|
|
95
|
+
downProj: GPUBuffer | WeightBuffer | Float32Array;
|
|
96
|
+
}
|
|
97
|
+
|
|
82
98
|
export interface PerLayerInputWeights {
|
|
83
99
|
embedTokensPerLayer: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
84
100
|
embedTokensPerLayerSplit?: (GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null)[] | null;
|
|
@@ -120,6 +136,41 @@ export interface LoadOptions {
|
|
|
120
136
|
verifyHashes: boolean;
|
|
121
137
|
}
|
|
122
138
|
|
|
139
|
+
export type LoaderLoadTimingPhase =
|
|
140
|
+
| 'preflight'
|
|
141
|
+
| 'tensorLocations'
|
|
142
|
+
| 'embeddings'
|
|
143
|
+
| 'layers'
|
|
144
|
+
| 'finalWeights'
|
|
145
|
+
| 'cleanup';
|
|
146
|
+
|
|
147
|
+
export interface LoaderLayerLoadTiming {
|
|
148
|
+
count: number | null;
|
|
149
|
+
totalMs: number | null;
|
|
150
|
+
meanMs: number | null;
|
|
151
|
+
maxMs: number | null;
|
|
152
|
+
maxLayer: number | null;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export interface LoaderLoadTiming {
|
|
156
|
+
schemaVersion: 1;
|
|
157
|
+
source: 'doppler-loader';
|
|
158
|
+
modelId: string | null;
|
|
159
|
+
status: 'running' | 'complete' | 'failed';
|
|
160
|
+
customShardLoader: boolean;
|
|
161
|
+
byteAccountingMode: 'full-shard-progress' | 'custom-loader-progress-unavailable';
|
|
162
|
+
totalBytes: number | null;
|
|
163
|
+
totalShards: number | null;
|
|
164
|
+
bytesLoaded: number;
|
|
165
|
+
shardsLoaded: number;
|
|
166
|
+
bytesPerSecond: number | null;
|
|
167
|
+
phasesMs: Record<LoaderLoadTimingPhase, number | null>;
|
|
168
|
+
layers: LoaderLayerLoadTiming;
|
|
169
|
+
totalMs: number | null;
|
|
170
|
+
failedPhase: LoaderLoadTimingPhase | null;
|
|
171
|
+
error: string | null;
|
|
172
|
+
}
|
|
173
|
+
|
|
123
174
|
/**
|
|
124
175
|
* Shard load priority.
|
|
125
176
|
*/
|
|
@@ -139,6 +190,7 @@ export interface CustomShardLoaderOptions {
|
|
|
139
190
|
verify?: boolean;
|
|
140
191
|
loadShardRange?: CustomShardRangeLoader;
|
|
141
192
|
streamShardRange?: CustomShardStreamLoader;
|
|
193
|
+
loadAuxiliaryFile?: CustomAuxiliaryFileLoader | null;
|
|
142
194
|
}
|
|
143
195
|
|
|
144
196
|
/**
|
|
@@ -174,6 +226,10 @@ export type CustomShardStreamLoader = (
|
|
|
174
226
|
options?: CustomShardStreamOptions
|
|
175
227
|
) => AsyncIterable<Uint8Array>;
|
|
176
228
|
|
|
229
|
+
export type CustomAuxiliaryFileLoader = (
|
|
230
|
+
path: string
|
|
231
|
+
) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
|
|
232
|
+
|
|
177
233
|
/**
|
|
178
234
|
* Loader statistics
|
|
179
235
|
*/
|
|
@@ -185,6 +241,7 @@ export interface LoaderStats {
|
|
|
185
241
|
layersLoaded: number;
|
|
186
242
|
expertsLoaded: number;
|
|
187
243
|
gpuBuffers: number;
|
|
244
|
+
loadTiming?: LoaderLoadTiming | null;
|
|
188
245
|
}
|
|
189
246
|
|
|
190
247
|
/**
|
|
@@ -37,6 +37,14 @@ export declare function getLargeWeightConfig(): LargeWeightConfigSchema;
|
|
|
37
37
|
*/
|
|
38
38
|
export declare function getLargeWeightMaxBytes(): number | null;
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Runtime large-weight overrides replace manifest overrides when present.
|
|
42
|
+
*/
|
|
43
|
+
export declare function resolveLargeWeightOverrides(
|
|
44
|
+
manifestOverrides: string[] | null | undefined,
|
|
45
|
+
runtimeOverrides: string[] | null | undefined
|
|
46
|
+
): string[] | null;
|
|
47
|
+
|
|
40
48
|
/**
|
|
41
49
|
* Estimate GPU memory required for a matmul weight after dequantization.
|
|
42
50
|
*
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { loadTensorsFromStore } from '../storage/shard-manager.js';
|
|
2
2
|
import { parseTensorMap } from '../formats/rdrr/index.js';
|
|
3
|
+
import {
|
|
4
|
+
assertFunctionalDescriptorManifest,
|
|
5
|
+
getFunctionalDescriptorManifest,
|
|
6
|
+
isFunctionalDescriptorDtype,
|
|
7
|
+
} from '../formats/rdrr/functional-descriptor.js';
|
|
3
8
|
import { log, trace as debugTrace } from '../debug/index.js';
|
|
4
9
|
|
|
5
10
|
function normalizeLocationSpans(spans, name, sourceLabel) {
|
|
@@ -24,6 +29,22 @@ function normalizeLocationSpans(spans, name, sourceLabel) {
|
|
|
24
29
|
});
|
|
25
30
|
}
|
|
26
31
|
|
|
32
|
+
function normalizeDescriptorManifest(info, name, sourceLabel) {
|
|
33
|
+
const descriptorManifest = getFunctionalDescriptorManifest(info);
|
|
34
|
+
if (descriptorManifest == null) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
return assertFunctionalDescriptorManifest(
|
|
38
|
+
descriptorManifest,
|
|
39
|
+
`Tensor "${name}" ${sourceLabel} descriptorManifest`
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function normalizeShardIndex(info, fallback = undefined) {
|
|
44
|
+
const shardIndex = info?.shardIndex ?? info?.shard;
|
|
45
|
+
return typeof shardIndex === 'number' ? shardIndex : fallback;
|
|
46
|
+
}
|
|
47
|
+
|
|
27
48
|
/**
|
|
28
49
|
* Builds the tensor name-to-location map from the manifest.
|
|
29
50
|
*
|
|
@@ -101,10 +122,14 @@ export async function buildTensorLocations(manifest, options = {}) {
|
|
|
101
122
|
if (!info.role) {
|
|
102
123
|
throw new Error(`Tensor "${name}" missing role in tensors.json`);
|
|
103
124
|
}
|
|
125
|
+
const descriptorManifest = normalizeDescriptorManifest(info, name, 'tensors.json');
|
|
126
|
+
if (isFunctionalDescriptorDtype(info.dtype) && descriptorManifest == null) {
|
|
127
|
+
throw new Error(`Tensor "${name}" missing descriptorManifest in tensors.json`);
|
|
128
|
+
}
|
|
104
129
|
locations.set(name, {
|
|
105
|
-
shardIndex: info
|
|
130
|
+
shardIndex: normalizeShardIndex(info),
|
|
106
131
|
offset: info.offset,
|
|
107
|
-
size: info.size,
|
|
132
|
+
size: info.size ?? 0,
|
|
108
133
|
shape: info.shape,
|
|
109
134
|
dtype: info.dtype,
|
|
110
135
|
role: info.role,
|
|
@@ -112,7 +137,9 @@ export async function buildTensorLocations(manifest, options = {}) {
|
|
|
112
137
|
spans: normalizeLocationSpans(info.spans, name, 'tensors.json'),
|
|
113
138
|
layout: info.layout,
|
|
114
139
|
originalShape: info.originalShape,
|
|
140
|
+
storage: info.storage,
|
|
115
141
|
sourceTransform: info.sourceTransform,
|
|
142
|
+
descriptorManifest,
|
|
116
143
|
});
|
|
117
144
|
}
|
|
118
145
|
const resolvedSource = trace.find((entry) => entry.outcome === 'resolved')?.source ?? 'unknown';
|
|
@@ -141,10 +168,14 @@ export async function buildTensorLocations(manifest, options = {}) {
|
|
|
141
168
|
if (!tensorInfo.role) {
|
|
142
169
|
throw new Error(`Tensor "${name}" missing role in manifest.tensors`);
|
|
143
170
|
}
|
|
171
|
+
const descriptorManifest = normalizeDescriptorManifest(tensorInfo, name, 'manifest.tensors');
|
|
172
|
+
if (isFunctionalDescriptorDtype(tensorInfo.dtype) && descriptorManifest == null) {
|
|
173
|
+
throw new Error(`Tensor "${name}" missing descriptorManifest in manifest.tensors`);
|
|
174
|
+
}
|
|
144
175
|
locations.set(name, {
|
|
145
|
-
shardIndex: tensorInfo
|
|
176
|
+
shardIndex: normalizeShardIndex(tensorInfo, isFunctionalDescriptorDtype(tensorInfo.dtype) ? undefined : 0),
|
|
146
177
|
offset: tensorInfo.offset,
|
|
147
|
-
size: tensorInfo.size,
|
|
178
|
+
size: tensorInfo.size ?? 0,
|
|
148
179
|
shape: tensorInfo.shape,
|
|
149
180
|
dtype: tensorInfo.dtype,
|
|
150
181
|
role: tensorInfo.role,
|
|
@@ -152,7 +183,9 @@ export async function buildTensorLocations(manifest, options = {}) {
|
|
|
152
183
|
spans: normalizeLocationSpans(tensorInfo.spans, name, 'manifest.tensors'),
|
|
153
184
|
layout: tensorInfo.layout,
|
|
154
185
|
originalShape: tensorInfo.originalShape,
|
|
186
|
+
storage: tensorInfo.storage,
|
|
155
187
|
sourceTransform: tensorInfo.sourceTransform,
|
|
188
|
+
descriptorManifest,
|
|
156
189
|
});
|
|
157
190
|
}
|
|
158
191
|
debugTrace.loader(`Tensor map: ${locations.size} tensors (inline)`);
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { TensorLocation } from '../loader-types.js';
|
|
2
|
+
import type { TensorLoadConfig, TensorLoadResult } from './tensor-loader.js';
|
|
3
|
+
|
|
4
|
+
export declare function loadFunctionalDescriptor(
|
|
5
|
+
shardData: Uint8Array & { descriptorShards?: Map<string, Uint8Array> },
|
|
6
|
+
location: TensorLocation,
|
|
7
|
+
name: string,
|
|
8
|
+
config?: TensorLoadConfig
|
|
9
|
+
): Promise<TensorLoadResult>;
|
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
import { acquireBuffer, uploadData } from '../../memory/buffer-pool.js';
|
|
2
|
+
import { createWeightBuffer } from '../../gpu/weight-buffer.js';
|
|
3
|
+
import { assertFunctionalDescriptorManifest } from '../../formats/rdrr/functional-descriptor.js';
|
|
4
|
+
import { log } from '../../debug/index.js';
|
|
5
|
+
|
|
6
|
+
// Reconstruct a functional_descriptor tensor from its component shards and
|
|
7
|
+
// upload the result to a GPU weight buffer. All reconstruction runs on CPU;
|
|
8
|
+
// the result is a row-major F32 (or F16 when hardware supports it) buffer.
|
|
9
|
+
//
|
|
10
|
+
// shardData must carry descriptorShards: Map<string, Uint8Array> keyed by the
|
|
11
|
+
// shard_file names declared in the descriptor manifest (kron, siren, sparse).
|
|
12
|
+
// location must carry descriptorManifest: the parsed manifoldgguf.v0.1 object.
|
|
13
|
+
|
|
14
|
+
// ============================================================================
|
|
15
|
+
// Coord-deterministic PRNG — must match tools/manifoldgguf-harness.js exactly.
|
|
16
|
+
// Algorithm identifier: coord_hash_normal_v1
|
|
17
|
+
// ============================================================================
|
|
18
|
+
|
|
19
|
+
function splitmix64(x) {
|
|
20
|
+
x = BigInt.asUintN(64, x + 0x9E3779B97F4A7C15n);
|
|
21
|
+
x = BigInt.asUintN(64, (x ^ (x >> 30n)) * 0xBF58476D1CE4E5B9n);
|
|
22
|
+
x = BigInt.asUintN(64, (x ^ (x >> 27n)) * 0x94D049BB133111EBn);
|
|
23
|
+
return x ^ (x >> 31n);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function coordUniform(seed, row, col) {
|
|
27
|
+
const s = BigInt(seed);
|
|
28
|
+
const r = BigInt(row);
|
|
29
|
+
const c = BigInt(col);
|
|
30
|
+
const z = splitmix64(
|
|
31
|
+
BigInt.asUintN(64, s ^ BigInt.asUintN(64, r * 0x9E3779B97F4A7C15n) ^ BigInt.asUintN(64, c * 0x6C62272E07BB0142n))
|
|
32
|
+
);
|
|
33
|
+
return Number(BigInt.asUintN(53, z)) / Number(1n << 53n);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function coordHashNormal(seed, row, col) {
|
|
37
|
+
const u1 = Math.max(coordUniform(seed, row, col), 1e-10);
|
|
38
|
+
const u2 = coordUniform(seed + 1, row, col);
|
|
39
|
+
return Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// ============================================================================
|
|
43
|
+
// PRNG component
|
|
44
|
+
// ============================================================================
|
|
45
|
+
|
|
46
|
+
function buildPRNG(prngSpec, rows, cols) {
|
|
47
|
+
if (prngSpec.algorithm !== 'coord_hash_normal_v1') {
|
|
48
|
+
throw new Error(
|
|
49
|
+
`[FunctionalDescriptor] Unknown PRNG algorithm "${prngSpec.algorithm}". ` +
|
|
50
|
+
'Only coord_hash_normal_v1 is supported in this runtime version.'
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
const { seed, learned_scale: scale } = prngSpec;
|
|
54
|
+
const out = new Float32Array(rows * cols);
|
|
55
|
+
for (let r = 0; r < rows; r++) {
|
|
56
|
+
for (let c = 0; c < cols; c++) {
|
|
57
|
+
out[r * cols + c] = scale * coordHashNormal(seed, r, c);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// ============================================================================
|
|
64
|
+
// Kronecker component
|
|
65
|
+
// Binary layout: [rank u32][a,b,c,d u32x4][A f32...][B f32...] per term
|
|
66
|
+
// ============================================================================
|
|
67
|
+
|
|
68
|
+
function blockUnpermute(P, a, b, c, d) {
|
|
69
|
+
const rows = a * b;
|
|
70
|
+
const cols = c * d;
|
|
71
|
+
const W = new Float32Array(rows * cols);
|
|
72
|
+
for (let ra = 0; ra < a; ra++) {
|
|
73
|
+
for (let rb = 0; rb < b; rb++) {
|
|
74
|
+
const i = ra * b + rb;
|
|
75
|
+
for (let cc = 0; cc < c; cc++) {
|
|
76
|
+
for (let cd = 0; cd < d; cd++) {
|
|
77
|
+
const j = cc * d + cd;
|
|
78
|
+
W[i * cols + j] = P[(ra * c + cc) * (b * d) + (rb * d + cd)];
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return W;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function deserializeKronecker(buf) {
|
|
87
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
88
|
+
const rank = view.getUint32(0, true);
|
|
89
|
+
let off = 4;
|
|
90
|
+
const factors = [];
|
|
91
|
+
for (let r = 0; r < rank; r++) {
|
|
92
|
+
const a = view.getUint32(off, true); off += 4;
|
|
93
|
+
const b = view.getUint32(off, true); off += 4;
|
|
94
|
+
const c = view.getUint32(off, true); off += 4;
|
|
95
|
+
const d = view.getUint32(off, true); off += 4;
|
|
96
|
+
const aLen = a * c;
|
|
97
|
+
const bLen = b * d;
|
|
98
|
+
const A = new Float32Array(buf.buffer, buf.byteOffset + off, aLen); off += aLen * 4;
|
|
99
|
+
const B = new Float32Array(buf.buffer, buf.byteOffset + off, bLen); off += bLen * 4;
|
|
100
|
+
factors.push({ a, b, c, d, A, B });
|
|
101
|
+
}
|
|
102
|
+
return factors;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function reconstructKronecker(factors, rows, cols) {
|
|
106
|
+
if (factors.length === 0) return new Float32Array(rows * cols);
|
|
107
|
+
const { a, b, c, d } = factors[0];
|
|
108
|
+
const p = a * c;
|
|
109
|
+
const q = b * d;
|
|
110
|
+
const P = new Float32Array(p * q);
|
|
111
|
+
for (const { A, B } of factors) {
|
|
112
|
+
for (let i = 0; i < p; i++) {
|
|
113
|
+
for (let j = 0; j < q; j++) {
|
|
114
|
+
P[i * q + j] += A[i] * B[j];
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return blockUnpermute(P, a, b, c, d);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// ============================================================================
|
|
122
|
+
// SIREN component
|
|
123
|
+
// Binary layout: [depth u32][inDim,outDim u32x2][W f32...][b f32...] per layer
|
|
124
|
+
// ============================================================================
|
|
125
|
+
|
|
126
|
+
function deserializeSIREN(buf) {
|
|
127
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
128
|
+
const depth = view.getUint32(0, true);
|
|
129
|
+
let off = 4;
|
|
130
|
+
const layers = [];
|
|
131
|
+
for (let l = 0; l < depth; l++) {
|
|
132
|
+
const inDim = view.getUint32(off, true); off += 4;
|
|
133
|
+
const outDim = view.getUint32(off, true); off += 4;
|
|
134
|
+
const W = new Float32Array(buf.buffer, buf.byteOffset + off, inDim * outDim); off += inDim * outDim * 4;
|
|
135
|
+
const b = new Float32Array(buf.buffer, buf.byteOffset + off, outDim); off += outDim * 4;
|
|
136
|
+
const isLast = l === depth - 1;
|
|
137
|
+
layers.push({ inDim, outDim, W, b, omega: l === 0 ? 30.0 : 1.0, isLast });
|
|
138
|
+
}
|
|
139
|
+
return layers;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function sirenForward(layers, rn, cn) {
|
|
143
|
+
let x = [rn, cn];
|
|
144
|
+
for (const { W, b, inDim, outDim, omega, isLast } of layers) {
|
|
145
|
+
const next = new Array(outDim);
|
|
146
|
+
for (let o = 0; o < outDim; o++) {
|
|
147
|
+
let s = b[o];
|
|
148
|
+
for (let i = 0; i < inDim; i++) s += W[o * inDim + i] * x[i];
|
|
149
|
+
next[o] = isLast ? s : Math.sin(omega * s);
|
|
150
|
+
}
|
|
151
|
+
x = next;
|
|
152
|
+
}
|
|
153
|
+
return x[0];
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function reconstructSIREN(layers, rows, cols) {
|
|
157
|
+
const out = new Float32Array(rows * cols);
|
|
158
|
+
for (let r = 0; r < rows; r++) {
|
|
159
|
+
const rn = rows > 1 ? (r / (rows - 1)) * 2 - 1 : 0;
|
|
160
|
+
for (let c = 0; c < cols; c++) {
|
|
161
|
+
const cn = cols > 1 ? (c / (cols - 1)) * 2 - 1 : 0;
|
|
162
|
+
out[r * cols + c] = sirenForward(layers, rn, cn);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return out;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function cropMatrix(matrix, rows, cols, cropRows, cropCols) {
|
|
169
|
+
if (rows === cropRows && cols === cropCols) {
|
|
170
|
+
return matrix;
|
|
171
|
+
}
|
|
172
|
+
const out = new Float32Array(cropRows * cropCols);
|
|
173
|
+
for (let r = 0; r < cropRows; r += 1) {
|
|
174
|
+
out.set(
|
|
175
|
+
matrix.subarray(r * cols, (r * cols) + cropCols),
|
|
176
|
+
r * cropCols
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
return out;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function normalizeShape2(value, label) {
|
|
183
|
+
if (!Array.isArray(value) || value.length !== 2) {
|
|
184
|
+
throw new Error(`[FunctionalDescriptor] ${label} must be [rows, cols].`);
|
|
185
|
+
}
|
|
186
|
+
const rows = Number(value[0]);
|
|
187
|
+
const cols = Number(value[1]);
|
|
188
|
+
if (!Number.isInteger(rows) || rows <= 0 || !Number.isInteger(cols) || cols <= 0) {
|
|
189
|
+
throw new Error(`[FunctionalDescriptor] ${label} has invalid shape [${value.join(',')}].`);
|
|
190
|
+
}
|
|
191
|
+
return [rows, cols];
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function getDescriptorShardBytes(shards) {
|
|
195
|
+
let total = 0;
|
|
196
|
+
for (const bytes of shards.values()) {
|
|
197
|
+
total += bytes.byteLength;
|
|
198
|
+
}
|
|
199
|
+
return total;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function assertOptionalIntegerEquals(value, expected, label, name) {
|
|
203
|
+
if (value === undefined) {
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
const parsed = Number(value);
|
|
207
|
+
if (!Number.isInteger(parsed) || parsed !== expected) {
|
|
208
|
+
throw new Error(
|
|
209
|
+
`[FunctionalDescriptor] ${label} for "${name}" must equal ${expected}, got ${value}.`
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function assertRuntimeProofGates(manifest, name, descriptorBytes, denseF16Bytes) {
|
|
215
|
+
assertOptionalIntegerEquals(manifest.descriptor_bytes, descriptorBytes, 'descriptor_bytes', name);
|
|
216
|
+
assertOptionalIntegerEquals(manifest.dense_f16_bytes, denseF16Bytes, 'dense_f16_bytes', name);
|
|
217
|
+
|
|
218
|
+
const gate = manifest.proof_status_gate;
|
|
219
|
+
const proofStatus = typeof manifest.proof_status === 'string'
|
|
220
|
+
? manifest.proof_status.trim().toLowerCase()
|
|
221
|
+
: null;
|
|
222
|
+
const sensitivityGate = typeof gate?.sensitivity === 'string'
|
|
223
|
+
? gate.sensitivity.trim().toLowerCase()
|
|
224
|
+
: null;
|
|
225
|
+
const compressionGate = typeof gate?.compression === 'string'
|
|
226
|
+
? gate.compression.trim().toLowerCase()
|
|
227
|
+
: null;
|
|
228
|
+
|
|
229
|
+
if (proofStatus === 'passed' && sensitivityGate !== 'passed') {
|
|
230
|
+
throw new Error(
|
|
231
|
+
`[FunctionalDescriptor] proof_status is "passed" for "${name}" but proof_status_gate.sensitivity is not "passed".`
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
if (proofStatus === 'passed' && compressionGate !== 'passed') {
|
|
235
|
+
throw new Error(
|
|
236
|
+
`[FunctionalDescriptor] proof_status is "passed" for "${name}" but proof_status_gate.compression is not "passed".`
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
if (compressionGate === 'passed' && descriptorBytes >= denseF16Bytes) {
|
|
240
|
+
throw new Error(
|
|
241
|
+
`[FunctionalDescriptor] compression proof gate failed for "${name}": ` +
|
|
242
|
+
`descriptor_bytes=${descriptorBytes} must be lower than dense_f16_bytes=${denseF16Bytes}.`
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// ============================================================================
|
|
248
|
+
// Sparse component (coo_v1)
|
|
249
|
+
// Binary layout: [nnz u32][rowIdx i32...][colIdx i32...][values f32...]
|
|
250
|
+
// ============================================================================
|
|
251
|
+
|
|
252
|
+
function deserializeSparse(buf) {
|
|
253
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
254
|
+
const nnz = view.getUint32(0, true);
|
|
255
|
+
let off = 4;
|
|
256
|
+
const rowIdx = new Int32Array(buf.buffer, buf.byteOffset + off, nnz); off += nnz * 4;
|
|
257
|
+
const colIdx = new Int32Array(buf.buffer, buf.byteOffset + off, nnz); off += nnz * 4;
|
|
258
|
+
const values = new Float32Array(buf.buffer, buf.byteOffset + off, nnz);
|
|
259
|
+
return { rowIdx, colIdx, values };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function applySparse(out, sparse, cols) {
|
|
263
|
+
const { rowIdx, colIdx, values } = sparse;
|
|
264
|
+
for (let k = 0; k < values.length; k++) {
|
|
265
|
+
out[rowIdx[k] * cols + colIdx[k]] += values[k];
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// ============================================================================
|
|
270
|
+
// Public entry point
|
|
271
|
+
// ============================================================================
|
|
272
|
+
|
|
273
|
+
export async function loadFunctionalDescriptor(shardData, location, name) {
|
|
274
|
+
if (!location.descriptorManifest) {
|
|
275
|
+
throw new Error(
|
|
276
|
+
`[FunctionalDescriptor] location.descriptorManifest is required for tensor "${name}". ` +
|
|
277
|
+
'The layer loader must parse the manifoldgguf manifest and attach it before calling loadTensorToGPU.'
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
const manifest = assertFunctionalDescriptorManifest(
|
|
281
|
+
location.descriptorManifest,
|
|
282
|
+
`FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptorManifest`
|
|
283
|
+
);
|
|
284
|
+
|
|
285
|
+
const shards = shardData?.descriptorShards;
|
|
286
|
+
if (!shards || typeof shards.get !== 'function') {
|
|
287
|
+
throw new Error(
|
|
288
|
+
`[FunctionalDescriptor] shardData.descriptorShards (Map) is required for tensor "${name}". ` +
|
|
289
|
+
'The shard manager must load all component shard files before dispatch.'
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
const { components } = manifest;
|
|
294
|
+
const [rows, cols] = normalizeShape2(
|
|
295
|
+
manifest.padded_shape ?? manifest.slice_shape,
|
|
296
|
+
'descriptor reconstruction shape'
|
|
297
|
+
);
|
|
298
|
+
const [cropRows, cropCols] = normalizeShape2(
|
|
299
|
+
manifest.crop_shape ?? location.shape ?? manifest.slice_shape,
|
|
300
|
+
'descriptor crop shape'
|
|
301
|
+
);
|
|
302
|
+
if (cropRows > rows || cropCols > cols) {
|
|
303
|
+
throw new Error(
|
|
304
|
+
`[FunctionalDescriptor] crop shape [${cropRows},${cropCols}] exceeds descriptor shape [${rows},${cols}] for "${name}".`
|
|
305
|
+
);
|
|
306
|
+
}
|
|
307
|
+
const descriptorBytes = getDescriptorShardBytes(shards);
|
|
308
|
+
const denseF16Bytes = cropRows * cropCols * 2;
|
|
309
|
+
assertRuntimeProofGates(manifest, name, descriptorBytes, denseF16Bytes);
|
|
310
|
+
|
|
311
|
+
log.debug('FunctionalDescriptor', `Reconstructing "${name}" [${rows}x${cols}]`);
|
|
312
|
+
|
|
313
|
+
const reconstructed = new Float32Array(rows * cols);
|
|
314
|
+
|
|
315
|
+
// PRNG substrate
|
|
316
|
+
const prngSpec = components.prng_substrate;
|
|
317
|
+
const prng = buildPRNG(prngSpec, rows, cols);
|
|
318
|
+
for (let i = 0; i < reconstructed.length; i++) reconstructed[i] += prng[i];
|
|
319
|
+
|
|
320
|
+
// Kronecker sum
|
|
321
|
+
const kronSpec = components.kronecker_sum;
|
|
322
|
+
const kronShard = shards.get(kronSpec.shard_file);
|
|
323
|
+
if (!kronShard) {
|
|
324
|
+
throw new Error(`[FunctionalDescriptor] Kron shard "${kronSpec.shard_file}" not found in descriptorShards for "${name}".`);
|
|
325
|
+
}
|
|
326
|
+
const kronFactors = deserializeKronecker(
|
|
327
|
+
kronShard instanceof Uint8Array ? kronShard : new Uint8Array(kronShard)
|
|
328
|
+
);
|
|
329
|
+
const kron = reconstructKronecker(kronFactors, rows, cols);
|
|
330
|
+
for (let i = 0; i < reconstructed.length; i++) reconstructed[i] += kron[i];
|
|
331
|
+
|
|
332
|
+
// SIREN INR
|
|
333
|
+
const sirenSpec = components.coordinate_inr;
|
|
334
|
+
const sirenShard = shards.get(sirenSpec.shard_file);
|
|
335
|
+
if (!sirenShard) {
|
|
336
|
+
throw new Error(`[FunctionalDescriptor] SIREN shard "${sirenSpec.shard_file}" not found in descriptorShards for "${name}".`);
|
|
337
|
+
}
|
|
338
|
+
const sirenLayers = deserializeSIREN(
|
|
339
|
+
sirenShard instanceof Uint8Array ? sirenShard : new Uint8Array(sirenShard)
|
|
340
|
+
);
|
|
341
|
+
const siren = reconstructSIREN(sirenLayers, rows, cols);
|
|
342
|
+
for (let i = 0; i < reconstructed.length; i++) reconstructed[i] += siren[i];
|
|
343
|
+
|
|
344
|
+
// Sparse residuals
|
|
345
|
+
const sparseSpec = components.sparse_outliers;
|
|
346
|
+
const sparseShard = shards.get(sparseSpec.shard_file);
|
|
347
|
+
if (!sparseShard) {
|
|
348
|
+
throw new Error(`[FunctionalDescriptor] Sparse shard "${sparseSpec.shard_file}" not found in descriptorShards for "${name}".`);
|
|
349
|
+
}
|
|
350
|
+
const sparse = deserializeSparse(
|
|
351
|
+
sparseShard instanceof Uint8Array ? sparseShard : new Uint8Array(sparseShard)
|
|
352
|
+
);
|
|
353
|
+
applySparse(reconstructed, sparse, cols);
|
|
354
|
+
|
|
355
|
+
log.debug('FunctionalDescriptor', `"${name}" reconstructed, uploading to GPU`);
|
|
356
|
+
const out = cropMatrix(reconstructed, rows, cols, cropRows, cropCols);
|
|
357
|
+
|
|
358
|
+
// Upload as dense F32 weight buffer
|
|
359
|
+
const gpuBuf = acquireBuffer(out.byteLength, GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST);
|
|
360
|
+
uploadData(gpuBuf, out);
|
|
361
|
+
|
|
362
|
+
const data = createWeightBuffer(gpuBuf, 'f32', 'row', [cropRows, cropCols], name, null, {
|
|
363
|
+
storageType: 'functional_descriptor',
|
|
364
|
+
descriptorHash: manifest.descriptor_hash,
|
|
365
|
+
descriptorBytes,
|
|
366
|
+
denseF16Bytes,
|
|
367
|
+
compressionRatio: descriptorBytes > 0 ? denseF16Bytes / descriptorBytes : null,
|
|
368
|
+
proofStatus: manifest.proof_status ?? null,
|
|
369
|
+
proofStatusGate: manifest.proof_status_gate ?? null,
|
|
370
|
+
descriptorShape: [rows, cols],
|
|
371
|
+
cropShape: [cropRows, cropCols],
|
|
372
|
+
});
|
|
373
|
+
return {
|
|
374
|
+
data,
|
|
375
|
+
allocatedBuffers: [gpuBuf],
|
|
376
|
+
};
|
|
377
|
+
}
|
|
@@ -72,6 +72,16 @@ export declare function getWeightLayout(
|
|
|
72
72
|
config: TensorLoadConfig
|
|
73
73
|
): WeightLayout;
|
|
74
74
|
|
|
75
|
+
export declare function isLiteRTAffineInt4FusedEligible(
|
|
76
|
+
location: TensorLocation,
|
|
77
|
+
config: Partial<TensorLoadConfig> | null
|
|
78
|
+
): boolean;
|
|
79
|
+
|
|
80
|
+
export declare function isW4A16FusedEligible(
|
|
81
|
+
location: TensorLocation,
|
|
82
|
+
config: Partial<TensorLoadConfig> | null
|
|
83
|
+
): boolean;
|
|
84
|
+
|
|
75
85
|
/**
|
|
76
86
|
* Convert BF16 data to F32 on CPU.
|
|
77
87
|
*/
|
|
@@ -91,6 +101,20 @@ export declare function loadQ4KFused(
|
|
|
91
101
|
name: string
|
|
92
102
|
): Promise<TensorLoadResult>;
|
|
93
103
|
|
|
104
|
+
export declare function loadLiteRTInt4Fused(
|
|
105
|
+
shardData: Uint8Array,
|
|
106
|
+
location: TensorLocation,
|
|
107
|
+
name: string,
|
|
108
|
+
config?: TensorLoadConfig | null
|
|
109
|
+
): Promise<TensorLoadResult>;
|
|
110
|
+
|
|
111
|
+
export declare function loadW4A16Fused(
|
|
112
|
+
shardData: Uint8Array,
|
|
113
|
+
location: TensorLocation,
|
|
114
|
+
name: string,
|
|
115
|
+
config?: TensorLoadConfig | null
|
|
116
|
+
): Promise<TensorLoadResult>;
|
|
117
|
+
|
|
94
118
|
/**
|
|
95
119
|
* Load Q4K tensor to GPU with dequantization.
|
|
96
120
|
*/
|
|
@@ -131,6 +155,16 @@ export declare function loadFloat(
|
|
|
131
155
|
config: TensorLoadConfig
|
|
132
156
|
): Promise<TensorLoadResult>;
|
|
133
157
|
|
|
158
|
+
/**
|
|
159
|
+
* Load W4A16 packed tensor through the reference dequantization path.
|
|
160
|
+
*/
|
|
161
|
+
export declare function loadW4A16Dequant(
|
|
162
|
+
shardData: Uint8Array,
|
|
163
|
+
location: TensorLocation,
|
|
164
|
+
name: string,
|
|
165
|
+
config: TensorLoadConfig
|
|
166
|
+
): Promise<TensorLoadResult>;
|
|
167
|
+
|
|
134
168
|
/**
|
|
135
169
|
* Load tensor data to GPU based on dtype.
|
|
136
170
|
*
|
|
@@ -158,5 +192,6 @@ export declare function loadTensorToGPU(
|
|
|
158
192
|
*/
|
|
159
193
|
export declare function loadTensorToCPU(
|
|
160
194
|
shardData: Uint8Array,
|
|
161
|
-
location: TensorLocation
|
|
195
|
+
location: TensorLocation,
|
|
196
|
+
name?: string | null
|
|
162
197
|
): Float32Array | Uint8Array;
|