doppler-gpu 0.5.1 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -577
- package/LICENSE +21 -201
- package/README.md +165 -222
- package/models/catalog.json +241 -33
- package/package.json +106 -87
- package/src/capsule-runtime.d.ts +25 -0
- package/src/capsule-runtime.js +24 -0
- package/src/capsule.d.ts +8 -0
- package/src/capsule.js +4 -0
- package/src/cli/argument-contract.js +200 -0
- package/src/cli/browser-report-output.js +24 -0
- package/src/cli/cli-model-resolution.js +2 -2
- package/src/cli/cli-output.js +10 -0
- package/src/cli/commands/bundle.js +828 -0
- package/src/cli/doppler-cli.js +50 -1457
- package/src/cli/doppler-serve.js +18 -244
- package/src/cli/output/manifest.js +368 -0
- package/src/cli/runtime-profiles.js +1 -1
- package/src/cli/serve/capsule-handler.d.ts +33 -0
- package/src/cli/serve/capsule-handler.js +105 -0
- package/src/cli/serve/capsule-http.js +102 -0
- package/src/cli/serve/receipt.js +269 -0
- package/src/client/capsule-host.browser.js +2 -0
- package/src/client/capsule-host.d.ts +5 -0
- package/src/client/capsule-host.js +2 -0
- package/src/client/doppler-api.browser.js +42 -0
- package/src/client/doppler-api.d.ts +38 -1
- package/src/client/doppler-api.js +41 -1
- package/src/client/doppler-registry.js +15 -7
- package/src/client/electron/index.d.ts +4 -0
- package/src/client/electron/index.js +13 -0
- package/src/client/electron/ipc-contract.d.ts +9 -0
- package/src/client/electron/ipc-contract.js +56 -0
- package/src/client/electron/release-state.d.ts +53 -0
- package/src/client/electron/release-state.js +425 -0
- package/src/client/electron/renderer-runtime.d.ts +19 -0
- package/src/client/electron/renderer-runtime.js +104 -0
- package/src/client/failure-taxonomy.js +1 -28
- package/src/client/fault-injection.js +0 -8
- package/src/client/inspection.js +1 -1
- package/src/client/model-host/index.d.ts +140 -0
- package/src/client/model-host/index.js +548 -0
- package/src/client/model-host/model-evidence.js +255 -0
- package/src/client/model-host/model-session.d.ts +323 -0
- package/src/client/{runtime → model-host}/model-session.js +202 -148
- package/src/client/provider.d.ts +18 -2
- package/src/client/provider.js +26 -3
- package/src/client/receipt.js +30 -72
- package/src/client/runtime/capsule-acquisition.d.ts +18 -0
- package/src/client/runtime/capsule-acquisition.js +103 -0
- package/src/client/runtime/capsule-adapter-execution.d.ts +17 -0
- package/src/client/runtime/capsule-adapter-execution.js +63 -0
- package/src/client/runtime/capsule-artifact-source.js +62 -0
- package/src/client/runtime/capsule-embedding.d.ts +29 -0
- package/src/client/runtime/capsule-embedding.js +76 -0
- package/src/client/runtime/capsule-forecast-program.d.ts +2 -0
- package/src/client/runtime/capsule-forecast-program.js +85 -0
- package/src/client/runtime/capsule-forecast.d.ts +17 -0
- package/src/client/runtime/capsule-forecast.js +31 -0
- package/src/client/runtime/capsule-operation-adapters.d.ts +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +69 -0
- package/src/client/runtime/capsule-operation-executor.d.ts +25 -0
- package/src/client/runtime/capsule-operation-executor.js +101 -0
- package/src/client/runtime/capsule-program-adapter.js +153 -0
- package/src/client/runtime/capsule-release-authorization.js +25 -0
- package/src/client/runtime/capsule-rerank.d.ts +50 -0
- package/src/client/runtime/capsule-rerank.js +105 -0
- package/src/client/runtime/capsule-session-execution.js +89 -0
- package/src/client/runtime/command-executor.d.ts +15 -0
- package/src/client/runtime/command-executor.js +128 -0
- package/src/client/runtime/composition-root.d.ts +77 -0
- package/src/client/runtime/composition-root.js +352 -0
- package/src/client/runtime/fetch-capsule-artifact-store.d.ts +10 -0
- package/src/client/runtime/fetch-capsule-artifact-store.js +25 -0
- package/src/client/runtime/index.d.ts +1 -117
- package/src/client/runtime/index.js +1 -433
- package/src/client/runtime/lora.d.ts +14 -0
- package/src/client/runtime/lora.js +19 -1
- package/src/client/runtime/model-manager.js +12 -0
- package/src/client/runtime/model-rerank.js +30 -0
- package/src/client/runtime/model-session.d.ts +1 -221
- package/src/client/runtime/model-source.d.ts +9 -0
- package/src/client/runtime/model-source.js +46 -17
- package/src/client/runtime/node-quickstart-cache.js +1 -1
- package/src/client/runtime/resolution-policy.d.ts +20 -0
- package/src/client/runtime/resolution-policy.js +84 -0
- package/src/client/runtime/resource-binder.d.ts +24 -0
- package/src/client/runtime/resource-binder.js +135 -0
- package/src/client/runtime/scoped-session.d.ts +10 -1
- package/src/client/runtime/scoped-session.js +20 -1
- package/src/client/runtime/session-controller.d.ts +19 -0
- package/src/client/runtime/session-controller.js +92 -0
- package/src/client/runtime/source-runtime.js +1 -1
- package/src/client/runtime/target-selector.d.ts +9 -0
- package/src/client/runtime/target-selector.js +2 -0
- package/src/client/runtime/types.d.ts +1 -0
- package/src/client/runtime/verified-capsule-artifact-store.js +101 -0
- package/src/client/wrap-pipeline-handle.js +3 -9
- package/src/config/backward-registry-loader.js +1 -1
- package/src/config/capsule-adapter-policy.js +30 -0
- package/src/config/capsule-adapters.d.ts +21 -0
- package/src/config/capsule-adapters.js +40 -0
- package/src/config/capsule-adapters.json +9 -0
- package/src/config/capsule-loading.d.ts +2 -0
- package/src/config/capsule-loading.js +14 -0
- package/src/config/capsule-loading.schema.json +12 -0
- package/src/config/capsule-operation.d.ts +19 -0
- package/src/config/capsule-operation.js +63 -0
- package/src/config/capsule-operations.json +17 -0
- package/src/config/capsule-release-contract.d.ts +66 -0
- package/src/config/capsule-release-contract.js +197 -0
- package/src/config/capsule-release-events.d.ts +53 -0
- package/src/config/capsule-release-events.js +172 -0
- package/src/config/capsule-sequence-receipt.js +19 -0
- package/src/config/capsule-serve.d.ts +9 -0
- package/src/config/capsule-serve.js +28 -0
- package/src/config/capsule-serve.schema.json +14 -0
- package/src/config/capsule-signature.d.ts +6 -0
- package/src/config/capsule-signature.js +50 -0
- package/src/config/capsule-v2.d.ts +70 -0
- package/src/config/capsule-v2.js +431 -0
- package/src/config/capsule-v3.d.ts +19 -0
- package/src/config/capsule-v3.js +72 -0
- package/src/config/capsule.d.ts +22 -0
- package/src/config/capsule.js +52 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +1 -1
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +62 -4
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +1 -1
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +6 -6
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +6 -6
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +442 -0
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +3 -3
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +529 -0
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +529 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +6 -6
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +6 -6
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +6 -6
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +7 -7
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +6 -6
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-f16-af32.json +463 -0
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +60 -8
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +561 -0
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +558 -0
- package/src/config/dtype-transition-contract.js +40 -0
- package/src/config/embedding-contract.js +26 -0
- package/src/config/embedding-reference.js +85 -0
- package/src/config/evidence/source-intake-policy.json +1 -1
- package/src/config/execution-contract-check.d.ts +16 -73
- package/src/config/execution-contract-check.js +7 -429
- package/src/config/forecast-manifest.js +56 -0
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/heterogeneous-attention-contract.js +46 -0
- package/src/config/initial-execution-identity.d.ts +58 -0
- package/src/config/initial-execution-identity.js +331 -0
- package/src/config/kernel-path-loader.d.ts +17 -0
- package/src/config/kernel-path-loader.js +42 -8
- package/src/config/kernel-registry-contract.d.ts +25 -0
- package/src/config/kernel-registry-contract.js +46 -0
- package/src/config/kernels/kernel-ref-digests.js +66 -18
- package/src/config/kernels/registry.js +1 -1
- package/src/config/kernels/registry.json +13617 -11377
- package/src/config/lora-layouts.d.ts +10 -0
- package/src/config/lora-layouts.js +17 -0
- package/src/config/lora-layouts.json +9 -0
- package/src/config/manifest-inference-validation.js +575 -0
- package/src/config/merge/execution-patches.js +12 -0
- package/src/config/{merge-helpers.js → merge/kernel-path-policy.js} +0 -52
- package/src/config/merge/value-selection.js +38 -0
- package/src/config/merge.js +43 -11
- package/src/config/model-ir-v2.d.ts +57 -0
- package/src/config/model-ir-v2.js +258 -0
- package/src/config/model-ir.d.ts +49 -0
- package/src/config/model-ir.js +180 -0
- package/src/config/param-validator.js +1 -1
- package/src/config/platforms/amd-strix-halo-gfx1151.json +17 -0
- package/src/config/platforms/loader.js +2 -1
- package/src/config/production-release-evidence.d.ts +23 -0
- package/src/config/production-release-evidence.js +423 -0
- package/src/config/production-release.js +321 -0
- package/src/{client/doppler-registry.json → config/quickstart-registry.json} +7 -7
- package/src/config/required-inference-fields-contract-check.js +2 -2
- package/src/config/rerank-reference.d.ts +28 -0
- package/src/config/rerank-reference.js +89 -0
- package/src/config/revocation-policy.d.ts +65 -0
- package/src/config/revocation-policy.js +290 -0
- package/src/config/revocation-registry.json +11 -0
- package/src/config/revocation-updates.d.ts +68 -0
- package/src/config/revocation-updates.js +440 -0
- package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity-f32kv.json +30 -0
- package/src/config/runtime/profiles/muse-glimmer-30b-source-parity.json +110 -0
- package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -1
- package/src/config/runtime/profiles/qwen3-8-27b-source-parity.json +87 -0
- package/src/config/runtime-merge.js +1 -1
- package/src/config/runtime.js +1 -1
- package/src/config/schema/browser-suite-metrics.schema.d.ts +1 -0
- package/src/config/schema/browser-suite-metrics.schema.js +2 -0
- package/src/config/schema/conversion-report.schema.d.ts +3 -0
- package/src/config/schema/conversion-report.schema.js +29 -0
- package/src/config/schema/converter.schema.d.ts +1 -0
- package/src/config/schema/debug.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +6 -5
- package/src/config/schema/emulation.schema.js +1 -1
- package/src/config/schema/execution-v1.schema.d.ts +3 -0
- package/src/config/schema/execution-v1.schema.js +19 -0
- package/src/config/schema/harness.schema.d.ts +13 -0
- package/src/config/schema/harness.schema.js +6 -0
- package/src/config/schema/inference.schema.d.ts +21 -2
- package/src/config/schema/kernel-thresholds.schema.js +1 -1
- package/src/config/schema/manifest.schema.d.ts +26 -7
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/program-bundle.schema.d.ts +52 -6
- package/src/config/schema/program-bundle.schema.js +145 -10
- package/src/config/sequence-reference.d.ts +26 -0
- package/src/config/sequence-reference.js +100 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/config/target-plan.d.ts +108 -0
- package/src/config/target-plan.js +411 -0
- package/src/config/training-defaults.js +1 -1
- package/src/config/transforms/capability-transform-resolver.js +4 -105
- package/src/config/transforms/execution-graph/dtype.js +591 -0
- package/src/config/transforms/execution-graph/fusion.js +436 -0
- package/src/config/transforms/execution-graph/validation.js +414 -0
- package/src/config/transforms/execution-graph-transforms.js +13 -1419
- package/src/converter/artifact-identity.js +475 -0
- package/src/converter/conversion-plan.js +1 -1
- package/src/converter/core.js +94 -2096
- package/src/converter/forge-candidate-evaluation.js +271 -0
- package/src/converter/forge-qualification.js +163 -0
- package/src/converter/forge-stages.js +800 -0
- package/src/converter/manifest-builder.js +683 -0
- package/src/converter/quantization-info.js +4 -1
- package/src/converter/quantizer.js +76 -13
- package/src/converter/shard-packer.d.ts +1 -1
- package/src/converter/shard-packer.js +2 -2
- package/src/converter/tensor-transform.js +640 -0
- package/src/converter/tokenizer-contract.js +243 -0
- package/src/converter/tokenizer-manifest.js +243 -0
- package/src/debug/history.js +0 -8
- package/src/debug/index.js +0 -10
- package/src/debug/tensor.js +1 -1
- package/src/debug/trace.js +0 -7
- package/src/experimental/adapters/adapter-manager.js +2 -0
- package/src/experimental/adapters/adapter-manifest.d.ts +6 -0
- package/src/experimental/adapters/adapter-manifest.js +11 -0
- package/src/experimental/adapters/artifact-contract.d.ts +2 -2
- package/src/experimental/adapters/artifact-contract.js +5 -2
- package/src/experimental/adapters/lora-loader.d.ts +10 -1
- package/src/experimental/adapters/lora-loader.js +110 -48
- package/src/{inference/pipelines → experimental}/audio/gemma4.js +9 -31
- package/src/experimental/bdpa/basis-decomposed-paged.d.ts +50 -0
- package/src/experimental/bdpa/basis-decomposed-paged.js +276 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +3 -3
- package/src/experimental/browser/browser-converter.js +11 -11
- package/src/experimental/diffusion/index.d.ts +1 -1
- package/src/experimental/diffusion/index.js +1 -1
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.d.ts +2 -2
- package/src/{inference/pipelines → experimental}/diffusion/pipeline.js +17 -17
- package/src/{inference/pipelines → experimental}/diffusion/text-encoder-gpu.js +21 -19
- package/src/experimental/diffusion/vae.js +596 -0
- package/src/experimental/distribution/shard-delivery/receipt.js +588 -0
- package/src/experimental/distribution/shard-delivery/retry.js +392 -0
- package/src/experimental/distribution/shard-delivery/transport.js +831 -0
- package/src/experimental/distribution/shard-delivery.js +4 -1709
- package/src/experimental/energy/index.d.ts +2 -2
- package/src/experimental/energy/index.js +2 -2
- package/src/{inference/pipelines → experimental}/energy/pipeline.js +13 -13
- package/src/{inference/pipelines → experimental}/energy/types.d.ts +2 -2
- package/src/experimental/logits/cpu-output.d.ts +27 -0
- package/src/{inference/pipelines/text/logits/utils.js → experimental/logits/cpu-output.js} +19 -27
- package/src/experimental/logits/cpu.d.ts +87 -0
- package/src/experimental/logits/cpu.js +95 -0
- package/src/experimental/orchestration/energy-row-head-pipeline.d.ts +103 -0
- package/src/{inference/pipelines/energy-head/row-head-pipeline.js → experimental/orchestration/energy-row-head-pipeline.js} +13 -13
- package/src/experimental/orchestration/index.d.ts +1 -1
- package/src/experimental/orchestration/index.js +1 -1
- package/src/experimental/training/artifacts.js +2 -2
- package/src/experimental/training/checkpoint.js +2 -2
- package/src/experimental/training/distillation/artifacts.js +1 -1
- package/src/experimental/training/distillation/dataset.js +2 -2
- package/src/experimental/training/distillation/eval.js +1 -1
- package/src/experimental/training/distillation/runtime.js +2 -2
- package/src/experimental/training/distillation/student-fixture/execution.js +166 -0
- package/src/experimental/training/distillation/student-fixture/model.js +353 -0
- package/src/experimental/training/distillation/student-fixture.js +2 -469
- package/src/experimental/training/lora/checkpoint.js +746 -0
- package/src/experimental/training/lora/execution.js +550 -0
- package/src/experimental/training/lora/plan.js +387 -0
- package/src/experimental/training/lora/recovery.js +418 -0
- package/src/experimental/training/lora-pipeline.js +7 -1949
- package/src/experimental/training/operator-artifacts.js +2 -2
- package/src/experimental/training/operator-command.js +4 -4
- package/src/experimental/training/runner/evaluation.js +62 -0
- package/src/experimental/training/runner/lifecycle.js +257 -0
- package/src/experimental/training/runner.js +5 -268
- package/src/experimental/training/suite/execution.js +879 -0
- package/src/experimental/training/suite/plan.js +855 -0
- package/src/experimental/training/suite.js +5 -1659
- package/src/experimental/training/workloads.d.ts +5 -5
- package/src/experimental/training/workloads.js +6 -6
- package/src/experimental/turboquant/codebook.js +470 -0
- package/src/formats/quantization-constants.d.ts +11 -0
- package/src/formats/quantization-constants.js +11 -0
- package/src/formats/rdrr/integrity.js +1 -1
- package/src/formats/rdrr/lowerings.js +0 -29
- package/src/formats/rdrr/merkle.js +1 -1
- package/src/formats/rdrr/tensor-config-validator.js +25 -5
- package/src/formats/rdrr/validation.js +3 -3
- package/src/gpu/command-recorder.d.ts +10 -1
- package/src/gpu/command-recorder.js +58 -14
- package/src/gpu/constant-buffer.js +29 -0
- package/src/gpu/device-state.js +44 -0
- package/src/gpu/device.js +11 -41
- package/src/gpu/kernel-runtime.js +2 -1
- package/src/gpu/kernels/activation-static-qdq.js +1 -1
- package/src/gpu/kernels/append_embedding.wgsl +14 -0
- package/src/gpu/kernels/attention/executors.js +960 -0
- package/src/gpu/kernels/attention/plan.js +472 -0
- package/src/gpu/kernels/attention.d.ts +4 -0
- package/src/gpu/kernels/attention.js +145 -1545
- package/src/gpu/kernels/attention_relative.wgsl +59 -0
- package/src/gpu/kernels/backward/adam.js +2 -1
- package/src/gpu/kernels/backward/attention_backward.js +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/bias_add_backward.js +1 -1
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +2 -1
- package/src/gpu/kernels/backward/conv2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/conv2d_backward.js +2 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/cross_entropy_backward.js +1 -1
- package/src/gpu/kernels/backward/embed_backward.js +2 -1
- package/src/gpu/kernels/backward/executor.d.ts +34 -0
- package/src/gpu/kernels/backward/executor.js +158 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +2 -1
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/gelu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/gelu_backward.js +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/groupnorm_backward.js +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/layernorm_backward.js +2 -1
- package/src/gpu/kernels/backward/matmul.js +133 -0
- package/src/gpu/kernels/backward/matmul_backward.d.ts +0 -1
- package/src/gpu/kernels/backward/matmul_backward.js +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/pixel_shuffle_backward.js +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rmsnorm_backward.js +2 -1
- package/src/gpu/kernels/backward/rope_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/rope_backward.js +2 -1
- package/src/gpu/kernels/backward/scale_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/scale_backward.js +2 -1
- package/src/gpu/kernels/backward/silu_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/silu_backward.js +1 -1
- package/src/gpu/kernels/backward/softmax_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/softmax_backward.js +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.d.ts +1 -1
- package/src/gpu/kernels/backward/upsample2d_backward.js +1 -1
- package/src/gpu/kernels/backward/utils.js +2 -392
- package/src/gpu/kernels/bind-group-validation.js +16 -0
- package/src/gpu/kernels/cast.js +2 -1
- package/src/gpu/kernels/check-finiteness.js +13 -49
- package/src/gpu/kernels/check-hot-vocab-stop.js +13 -39
- package/src/gpu/kernels/check-stop.js +14 -33
- package/src/gpu/kernels/check_finiteness.wgsl +33 -0
- package/src/gpu/kernels/check_hot_vocab_stop.wgsl +29 -0
- package/src/gpu/kernels/check_stop.wgsl +20 -0
- package/src/gpu/kernels/clamp.js +1 -1
- package/src/gpu/kernels/conv2d.js +1 -1
- package/src/gpu/kernels/cross_entropy_loss.js +1 -1
- package/src/gpu/kernels/depthwise_conv1d.js +1 -1
- package/src/gpu/kernels/depthwise_conv2d.js +1 -1
- package/src/gpu/kernels/dequant.js +7 -2
- package/src/gpu/kernels/diffusion-gemma-sampling.js +3 -5
- package/src/gpu/kernels/dispatch.js +12 -1
- package/src/gpu/kernels/embedding-pool.js +72 -0
- package/src/gpu/kernels/embedding_pool_f16.wgsl +38 -0
- package/src/gpu/kernels/embedding_pool_f32.wgsl +36 -0
- package/src/gpu/kernels/energy.js +2 -1
- package/src/gpu/kernels/forecast_output.wgsl +17 -0
- package/src/gpu/kernels/fused-gate-up-gelu.js +1 -1
- package/src/gpu/kernels/fused_ffn.js +10 -220
- package/src/gpu/kernels/fused_matmul_residual.js +2 -1
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +2 -1
- package/src/gpu/kernels/fused_normed_ffn_q4k.wgsl +208 -0
- package/src/gpu/kernels/gated-short-conv.js +11 -69
- package/src/gpu/kernels/gated_delta_conv.wgsl +66 -0
- package/src/gpu/kernels/gated_delta_conv_f16.wgsl +68 -0
- package/src/gpu/kernels/gated_delta_fused_decode.wgsl +173 -0
- package/src/gpu/kernels/gated_delta_fused_decode_f16.wgsl +175 -0
- package/src/gpu/kernels/gated_delta_recurrent.wgsl +165 -0
- package/src/gpu/kernels/gated_delta_recurrent_f16.wgsl +167 -0
- package/src/gpu/kernels/gated_short_conv.wgsl +47 -0
- package/src/gpu/kernels/gather.d.ts +1 -1
- package/src/gpu/kernels/gather.js +30 -6
- package/src/gpu/kernels/gather_bf16.wgsl +69 -0
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +12 -2
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/gemma4-route-expert.js +1 -1
- package/src/gpu/kernels/grouped_pointwise_conv2d.js +1 -1
- package/src/gpu/kernels/groupnorm.js +1 -1
- package/src/gpu/kernels/index.d.ts +38 -5
- package/src/gpu/kernels/index.js +18 -4
- package/src/gpu/kernels/instance_stats.wgsl +26 -0
- package/src/gpu/kernels/kernel-base.js +1 -1
- package/src/gpu/kernels/kernel-cache-lifecycle.d.ts +9 -0
- package/src/gpu/kernels/kernel-cache-lifecycle.js +26 -0
- package/src/gpu/kernels/kernel-configs.d.ts +10 -61
- package/src/gpu/kernels/kernel-configs.js +26 -54
- package/src/gpu/kernels/kernel-execution.js +141 -0
- package/src/gpu/kernels/kv-cache-write.js +4 -4
- package/src/gpu/kernels/kv-quantize.js +4 -1
- package/src/gpu/kernels/l2-normalize.js +36 -0
- package/src/gpu/kernels/l2_normalize.wgsl +53 -0
- package/src/gpu/kernels/layernorm.d.ts +4 -4
- package/src/gpu/kernels/layernorm.js +44 -7
- package/src/gpu/kernels/linear-attention-core.js +29 -449
- package/src/gpu/kernels/linear_attention.js +1 -1
- package/src/gpu/kernels/lm-head-argmax.js +2 -2
- package/src/gpu/kernels/lm-head-select-logits.js +2 -2
- package/src/gpu/kernels/logit-finalize.js +103 -0
- package/src/gpu/kernels/logit-merge.js +13 -125
- package/src/gpu/kernels/logit-scatter.js +40 -0
- package/src/gpu/kernels/logit_finalize_f16.wgsl +43 -0
- package/src/gpu/kernels/logit_finalize_f32.wgsl +41 -0
- package/src/gpu/kernels/logit_merge_geometric.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_max.wgsl +23 -0
- package/src/gpu/kernels/logit_merge_weighted.wgsl +23 -0
- package/src/gpu/kernels/logit_scatter_f16.wgsl +26 -0
- package/src/gpu/kernels/logit_scatter_f32.wgsl +24 -0
- package/src/gpu/kernels/matmul-dispatch.js +6 -1
- package/src/gpu/kernels/matmul-selection.js +3 -2
- package/src/gpu/kernels/matmul.js +4 -15
- package/src/gpu/kernels/matmul_bf16w_f32a.wgsl +93 -0
- package/src/gpu/kernels/matmul_gemv_subgroup_bf16w.wgsl +433 -0
- package/src/gpu/kernels/modulate.js +1 -1
- package/src/gpu/kernels/moe.js +3 -1
- package/src/gpu/kernels/moe_router_bias_add_f16_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f16_f32.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f16.wgsl +23 -0
- package/src/gpu/kernels/moe_router_bias_add_f32_f32.wgsl +21 -0
- package/src/gpu/kernels/patch_context.wgsl +29 -0
- package/src/gpu/kernels/pipeline-cache.js +3 -1
- package/src/gpu/kernels/pixel_shuffle.js +1 -1
- package/src/gpu/kernels/probe_shader_f16.wgsl +7 -0
- package/src/gpu/kernels/relu.js +1 -1
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/repeat_channels.js +1 -1
- package/src/gpu/kernels/residual.js +2 -2
- package/src/gpu/kernels/rmsnorm-pair.js +1 -1
- package/src/gpu/kernels/rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/rmsnorm-stats.js +14 -113
- package/src/gpu/kernels/rmsnorm.js +1 -1
- package/src/gpu/kernels/rmsnorm_serial.wgsl +19 -0
- package/src/gpu/kernels/rmsnorm_stats.wgsl +59 -0
- package/src/gpu/kernels/rmsnorm_stats_subgroups.wgsl +76 -0
- package/src/gpu/kernels/rope-precompute.js +160 -0
- package/src/gpu/kernels/rope-qk.js +1 -1
- package/src/gpu/kernels/rope.js +2 -2
- package/src/gpu/kernels/rope_precompute.wgsl +74 -0
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +36 -17
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/gpu/kernels/scale.js +1 -1
- package/src/gpu/kernels/shader-cache.js +8 -4
- package/src/gpu/kernels/shader-source-scope.js +68 -0
- package/src/gpu/kernels/silu.js +2 -1
- package/src/gpu/kernels/soft-embedding.js +3 -1
- package/src/gpu/kernels/softmax.js +4 -2
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +1 -1
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +1 -1
- package/src/gpu/kernels/split_qg.js +1 -1
- package/src/gpu/kernels/split_qkv.d.ts +2 -0
- package/src/gpu/kernels/split_qkv.js +18 -3
- package/src/gpu/kernels/split_qkv.wgsl +5 -2
- package/src/gpu/kernels/split_qkv_f16.wgsl +5 -2
- package/src/gpu/kernels/submit_probe.wgsl +6 -0
- package/src/gpu/kernels/transpose.js +1 -1
- package/src/gpu/kernels/turboquant-codebook.js +4 -535
- package/src/gpu/kernels/uniform-utils.js +7 -2
- package/src/gpu/kernels/upsample2d.js +1 -1
- package/src/gpu/kernels/utils.js +9 -330
- package/src/gpu/kernels/vision-average-pool.d.ts +13 -0
- package/src/gpu/kernels/vision-average-pool.js +68 -0
- package/src/gpu/kernels/vision-patch-embed.d.ts +17 -0
- package/src/gpu/kernels/vision-patch-embed.js +117 -0
- package/src/gpu/kernels/vision-position-embedding.d.ts +13 -0
- package/src/gpu/kernels/vision-position-embedding.js +79 -0
- package/src/gpu/kernels/vision-rope-2d.d.ts +16 -0
- package/src/gpu/kernels/vision-rope-2d.js +73 -0
- package/src/gpu/kernels/vision-spatial-merge.d.ts +19 -0
- package/src/gpu/kernels/vision-spatial-merge.js +100 -0
- package/src/gpu/kernels/vision_average_pool.wgsl +42 -0
- package/src/gpu/kernels/vision_patch_embed.wgsl +59 -0
- package/src/gpu/kernels/vision_patch_embed_f16w.wgsl +61 -0
- package/src/gpu/kernels/vision_position_embedding.wgsl +33 -0
- package/src/gpu/kernels/vision_position_embedding_f16.wgsl +35 -0
- package/src/gpu/kernels/vision_rope_2d.wgsl +60 -0
- package/src/gpu/kernels/vision_spatial_merge.wgsl +48 -0
- package/src/gpu/readback-buffer.js +20 -0
- package/src/gpu/submit-probe.js +2 -26
- package/src/gpu/weight-buffer.d.ts +1 -0
- package/src/gpu/weight-buffer.js +7 -0
- package/src/index.d.ts +12 -0
- package/src/index.js +2 -0
- package/src/inference/browser-harness/benchmark-execution.js +872 -0
- package/src/inference/browser-harness/benchmark-rerank.js +284 -0
- package/src/inference/browser-harness/decode-diagnostics.js +241 -0
- package/src/inference/browser-harness/diffusion-performance.js +102 -0
- package/src/inference/browser-harness/inference-execution.js +543 -0
- package/src/inference/{browser-harness-model-helpers.js → browser-harness/model-initialization.js} +14 -145
- package/src/inference/browser-harness/model-resolution.js +137 -0
- package/src/inference/browser-harness/report.js +338 -0
- package/src/inference/browser-harness/request.js +263 -0
- package/src/inference/browser-harness/runtime-config.d.ts +25 -0
- package/src/inference/browser-harness/runtime-config.js +255 -0
- package/src/inference/browser-harness/runtime-isolation.d.ts +13 -0
- package/src/inference/browser-harness/runtime-isolation.js +45 -0
- package/src/inference/browser-harness/runtime-report.d.ts +26 -0
- package/src/inference/browser-harness/runtime-report.js +117 -0
- package/src/inference/browser-harness/suite-summary.js +39 -0
- package/src/inference/browser-harness/text-evidence.js +694 -0
- package/src/inference/browser-harness/text-execution.js +757 -0
- package/src/inference/browser-harness/text-input.js +725 -0
- package/src/inference/{browser-harness-suite-helpers.js → browser-harness/timing-diagnostics.js} +1 -379
- package/src/inference/browser-harness-contract-helpers.js +1 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +8 -3
- package/src/inference/browser-harness-runtime-helpers.d.ts +3 -56
- package/src/inference/browser-harness-runtime-helpers.js +3 -423
- package/src/inference/browser-harness-text-helpers.js +31 -2199
- package/src/inference/browser-harness.d.ts +15 -2
- package/src/inference/browser-harness.js +38 -1952
- package/src/inference/kv-cache/base.js +67 -381
- package/src/inference/kv-cache/basis-decomposed-paged.d.ts +2 -50
- package/src/inference/kv-cache/basis-decomposed-paged.js +1 -276
- package/src/inference/kv-cache/quantized.d.ts +2 -2
- package/src/inference/kv-cache/quantized.js +4 -8
- package/src/inference/kv-cache/update.js +309 -0
- package/src/inference/model-checkpoint-evidence.d.ts +75 -0
- package/src/inference/model-checkpoint-evidence.js +264 -0
- package/src/inference/moe-router.js +36 -216
- package/src/inference/multi-model-network.js +5 -341
- package/src/inference/pipelines/audio/index.js +1 -20
- package/src/inference/pipelines/audio/mel.js +0 -38
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/diffusion/profile-timing.js +10 -0
- package/src/inference/pipelines/diffusion/runtime-resources.js +54 -0
- package/src/inference/pipelines/diffusion/sd3/execution.js +805 -0
- package/src/inference/pipelines/diffusion/sd3/plan.js +125 -0
- package/src/inference/pipelines/diffusion/sd3-transformer.js +44 -895
- package/src/inference/pipelines/diffusion/{vae.js → vae/execution.js} +46 -618
- package/src/inference/pipelines/diffusion/{helpers.js → weight-contract.js} +1 -68
- package/src/inference/pipelines/energy-head/row-head-pipeline.d.ts +1 -103
- package/src/inference/pipelines/factory.js +14 -3
- package/src/inference/pipelines/shader-scoped-pipeline.js +29 -0
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -124
- package/src/inference/pipelines/text/abort-contract.js +22 -0
- package/src/inference/pipelines/text/attention/attn-config.js +1 -16
- package/src/inference/pipelines/text/attention/gate-projection.js +30 -0
- package/src/inference/pipelines/text/attention/heterogeneous-contract.js +8 -0
- package/src/inference/pipelines/text/attention/interpreter/immediate.js +211 -0
- package/src/inference/pipelines/text/attention/interpreter/recorded.js +451 -0
- package/src/inference/pipelines/text/attention/interpreter.js +89 -439
- package/src/inference/pipelines/text/attention/plan.d.ts +1 -0
- package/src/inference/pipelines/text/attention/plan.js +17 -0
- package/src/inference/pipelines/text/attention/projection-executor.js +331 -0
- package/src/inference/pipelines/text/attention/projection-plan.js +259 -0
- package/src/inference/pipelines/text/attention/projections.js +20 -537
- package/src/inference/pipelines/text/attention/query-transform.js +11 -0
- package/src/inference/pipelines/text/attention/record.js +0 -6
- package/src/inference/pipelines/text/attention/recorded-health.js +21 -0
- package/src/inference/pipelines/text/attention/rope-observation.js +40 -0
- package/src/inference/pipelines/text/attention/run.js +0 -6
- package/src/inference/pipelines/text/attention/types.js +3 -31
- package/src/inference/pipelines/text/chat-format.d.ts +3 -0
- package/src/inference/pipelines/text/chat-format.js +42 -0
- package/src/inference/pipelines/text/config/normalization.js +729 -0
- package/src/inference/pipelines/text/config/validation.js +9 -0
- package/src/inference/pipelines/text/config.d.ts +12 -1
- package/src/inference/pipelines/text/config.js +14 -1228
- package/src/inference/pipelines/text/debug-utils/health-metrics.js +57 -0
- package/src/inference/pipelines/text/debug-utils/index.js +3 -8
- package/src/inference/pipelines/text/debug-utils/profiles.js +8 -0
- package/src/inference/pipelines/text/debug-utils/readback-decoding.js +24 -0
- package/src/inference/pipelines/text/debug-utils/tensor.js +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +1 -68
- package/src/inference/pipelines/text/embed.js +178 -241
- package/src/inference/pipelines/text/embedding-contract.d.ts +7 -0
- package/src/inference/pipelines/text/embedding-contract.js +21 -0
- package/src/inference/pipelines/text/embedding-extraction.js +225 -0
- package/src/inference/pipelines/text/embedding-normalization.js +39 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +0 -32
- package/src/inference/pipelines/text/execution-v1/plan.js +69 -0
- package/src/inference/pipelines/text/execution-v1/validation.js +233 -0
- package/src/inference/pipelines/text/execution-v1.js +7 -283
- package/src/inference/pipelines/text/execution.js +517 -0
- package/src/inference/pipelines/text/ffn/dense-executor.js +723 -0
- package/src/inference/pipelines/text/ffn/dense-plan.js +327 -0
- package/src/inference/pipelines/text/ffn/dense.js +6 -970
- package/src/inference/pipelines/text/ffn/sandwich.js +10 -9
- package/src/inference/pipelines/text/ffn/standard.js +2 -1
- package/src/inference/pipelines/text/generator/attention-lifecycle.js +24 -0
- package/src/inference/pipelines/text/generator/decode-runtime.js +985 -0
- package/src/inference/pipelines/text/generator/decode.js +964 -0
- package/src/inference/pipelines/text/generator/diffusion.js +607 -0
- package/src/inference/pipelines/text/generator/logits-config.js +63 -0
- package/src/inference/pipelines/text/generator/prefill-observation.js +29 -0
- package/src/inference/pipelines/text/generator/prefill-policy.js +56 -0
- package/src/inference/pipelines/text/generator/prefill-runtime.js +896 -0
- package/src/inference/pipelines/text/{generator-prefill-helpers.js → generator/prefix-embedding.js} +16 -121
- package/src/inference/pipelines/text/generator/prompt-input.js +47 -0
- package/src/inference/pipelines/text/generator/recovery.js +310 -0
- package/src/inference/pipelines/text/{generator-helpers.js → generator/session-context.js} +42 -110
- package/src/inference/pipelines/text/generator/text.js +448 -0
- package/src/inference/pipelines/text/generator/training.js +753 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +1 -1
- package/src/inference/pipelines/text/generator-logits-observation.js +48 -0
- package/src/inference/pipelines/text/generator-runtime.js +23 -154
- package/src/inference/pipelines/text/generator-steps.js +45 -980
- package/src/inference/pipelines/text/generator.js +179 -3552
- package/src/inference/pipelines/text/image-transcription.js +304 -0
- package/src/inference/pipelines/text/init-qkv-fusion.js +18 -9
- package/src/inference/pipelines/text/init-rope.js +37 -189
- package/src/inference/pipelines/text/init.d.ts +8 -3
- package/src/inference/pipelines/text/init.js +6 -216
- package/src/inference/pipelines/text/layer-execution-contract.js +97 -0
- package/src/inference/pipelines/text/layer-input-execution.js +435 -0
- package/src/inference/pipelines/text/layer-plan-gpu.js +13 -6
- package/src/inference/pipelines/text/layer-plan.d.ts +21 -5
- package/src/inference/pipelines/text/layer-plan.js +45 -3
- package/src/inference/pipelines/text/layer.js +52 -522
- package/src/inference/pipelines/text/lifecycle.js +839 -0
- package/src/inference/pipelines/text/linear-attention/plan.js +554 -0
- package/src/inference/pipelines/text/linear-attention-ab-fusion.js +2 -2
- package/src/inference/pipelines/text/linear-attention.d.ts +2 -2
- package/src/inference/pipelines/text/linear-attention.js +28 -534
- package/src/inference/pipelines/text/logits/cpu-output.d.ts +1 -0
- package/src/inference/pipelines/text/logits/cpu-output.js +5 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +1 -87
- package/src/inference/pipelines/text/logits/cpu.js +8 -95
- package/src/inference/pipelines/text/logits/gpu-executor.js +368 -0
- package/src/inference/pipelines/text/logits/gpu.d.ts +27 -0
- package/src/inference/pipelines/text/logits/gpu.js +116 -579
- package/src/inference/pipelines/text/logits/index.d.ts +2 -1
- package/src/inference/pipelines/text/logits/index.js +62 -102
- package/src/inference/pipelines/text/logits/output-transform.js +54 -0
- package/src/inference/pipelines/text/logits/plan.js +184 -0
- package/src/inference/pipelines/text/logits/readback.d.ts +6 -0
- package/src/inference/pipelines/text/logits/readback.js +9 -0
- package/src/inference/pipelines/text/logits/scale-policy.js +20 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/lora-apply.js +12 -4
- package/src/inference/pipelines/text/lora-types.d.ts +12 -0
- package/src/inference/pipelines/text/modality-token-contract.js +115 -0
- package/src/inference/pipelines/text/model-load.js +0 -20
- package/src/inference/pipelines/text/moe/expert-loading.js +9 -0
- package/src/inference/pipelines/text/moe/gpu-executor.js +513 -0
- package/src/inference/pipelines/text/moe/plan.js +47 -0
- package/src/inference/pipelines/text/moe-cpu.js +9 -135
- package/src/inference/pipelines/text/moe-gpu.js +21 -545
- package/src/inference/pipelines/text/normalization-contract.js +17 -0
- package/src/inference/pipelines/text/per-layer/materialize.js +736 -0
- package/src/inference/pipelines/text/per-layer/plan.js +362 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +4 -1037
- package/src/inference/pipelines/text/pipeline-load-timing.js +77 -0
- package/src/inference/pipelines/text/probes.js +20 -0
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/state.d.ts +8 -0
- package/src/inference/pipelines/text/state.js +1 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +88 -16
- package/src/inference/pipelines/text/types.d.ts +7 -3
- package/src/inference/pipelines/text/weight-loading-config.js +194 -0
- package/src/inference/pipelines/text.d.ts +2 -2
- package/src/inference/pipelines/text.js +46 -1633
- package/src/inference/pipelines/video/frame-sampler.js +0 -13
- package/src/inference/pipelines/video/index.js +0 -21
- package/src/inference/pipelines/vision/attention.js +59 -0
- package/src/inference/pipelines/vision/encoder.js +254 -369
- package/src/inference/pipelines/vision/gemma4.js +204 -318
- package/src/inference/pipelines/vision/glmocr-preprocess.js +273 -0
- package/src/inference/pipelines/vision/glmocr-rope.js +106 -0
- package/src/inference/pipelines/vision/glmocr.js +557 -0
- package/src/inference/pipelines/vision/image-preprocess.js +0 -27
- package/src/inference/pipelines/vision/index.js +8 -27
- package/src/inference/pipelines/vision/ops.js +11 -33
- package/src/inference/pipelines/vision/patch-embed.js +51 -145
- package/src/inference/pipelines/vision/spatial-merge.js +78 -0
- package/src/inference/pipelines/vision/weight-loading.js +79 -0
- package/src/inference/refactor-receipt.js +2 -2
- package/src/inference/rerank.js +182 -0
- package/src/inference/token-sampling.js +319 -0
- package/src/inference/tokenizer.js +1 -1
- package/src/inference/tokenizers/behavior-flags.js +6 -3
- package/src/inference/tokenizers/bundled/decoders.js +133 -0
- package/src/inference/tokenizers/bundled/registry.js +216 -0
- package/src/inference/tokenizers/bundled.js +23 -353
- package/src/loader/doppler-loader.d.ts +4 -0
- package/src/loader/doppler-loader.js +119 -930
- package/src/loader/embedding-limit-preflight.js +11 -4
- package/src/loader/embedding-loader.js +1 -3
- package/src/loader/gpu-tensor-materialization.js +20 -0
- package/src/loader/layer-loader.js +15 -119
- package/src/loader/layer-tensor-aliases.js +99 -0
- package/src/loader/layer-weight-shape.js +29 -0
- package/src/loader/load-timing.js +64 -0
- package/src/loader/loader-types.d.ts +3 -0
- package/src/loader/model-executor.js +819 -0
- package/src/loader/model-load-validation.js +15 -0
- package/src/loader/quantization-constants.d.ts +11 -29
- package/src/loader/quantization-constants.js +1 -2
- package/src/loader/shard-resolver.js +0 -14
- package/src/loader/tensor-load-error.js +14 -0
- package/src/loader/tensor-shard-indices.js +8 -0
- package/src/loader/tensors/bf16-loader.js +125 -0
- package/src/loader/tensors/functional-descriptor-loader.js +3 -375
- package/src/loader/tensors/load-executor.js +577 -0
- package/src/loader/tensors/source-transform/executor.js +123 -0
- package/src/loader/tensors/source-transform/plan.js +107 -0
- package/src/loader/tensors/source-transform.js +15 -264
- package/src/loader/tensors/tensor-loader.d.ts +4 -4
- package/src/loader/tensors/tensor-loader.js +7 -653
- package/src/memory/buffer-pool.js +7 -29
- package/src/rules/capability-transform-resolver.js +80 -0
- package/src/rules/execution-contract.d.ts +73 -0
- package/src/rules/execution-contract.js +429 -0
- package/src/rules/execution-rules-contract-check.js +1 -1
- package/src/rules/generated/rule-bundle.json +4305 -0
- package/src/rules/inference/capability-transforms.rules.json +3 -3
- package/src/rules/layer-pattern-contract-check.js +1 -1
- package/src/rules/rule-registry.js +70 -57
- package/src/storage/artifact-storage-context.js +2 -2
- package/src/storage/backend-selection.js +43 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/storage/distribution-transport.js +5 -0
- package/src/storage/download/integrity.d.ts +33 -0
- package/src/storage/download/integrity.js +130 -0
- package/src/storage/download/resume-inspection.d.ts +22 -0
- package/src/storage/download/resume-inspection.js +134 -0
- package/src/storage/download/retry.js +57 -0
- package/src/storage/download/state.js +131 -0
- package/src/storage/download/transport.js +99 -0
- package/src/storage/downloader.js +74 -453
- package/src/storage/model-cache.d.ts +1 -0
- package/src/storage/model-cache.js +51 -11
- package/src/storage/shard-manager.d.ts +5 -0
- package/src/storage/shard-manager.js +50 -369
- package/src/storage/shards/index.js +60 -0
- package/src/storage/shards/integrity.js +225 -0
- package/src/storage/shards/lifecycle.js +22 -0
- package/src/storage/source-artifact-store.js +14 -4
- package/src/tooling/boundary-evidence.d.ts +4 -4
- package/src/tooling/boundary-evidence.js +20 -20
- package/src/tooling/browser-command-runner.js +13 -6
- package/src/tooling/capsule-v2.d.ts +11 -0
- package/src/tooling/capsule-v2.js +58 -0
- package/src/tooling/capsule.js +20 -0
- package/src/tooling/command-api-constants.js +1 -1
- package/src/tooling/command-api-family-normalizers.js +163 -5
- package/src/tooling/command-api.d.ts +42 -3
- package/src/tooling/command-api.js +7 -2
- package/src/tooling/command-envelope.js +1 -1
- package/src/tooling/{command-api-helpers.js → command-request-normalization.js} +1 -1
- package/src/tooling/command-runner.html +49 -1
- package/src/tooling/command-validation.js +1 -24
- package/src/tooling/conversion-run-timing.js +24 -0
- package/src/tooling/diagnose-runner.js +11 -9
- package/src/tooling/distribution-shard-transport.js +11 -0
- package/src/tooling/execution-cost-ledger.d.ts +76 -2
- package/src/tooling/execution-cost-ledger.js +197 -15
- package/src/tooling/hf-registry/catalog-io.js +63 -0
- package/src/tooling/litert/package.js +863 -0
- package/src/tooling/litert-package-runtime.js +3 -849
- package/src/tooling/model-capsule-forge.js +478 -0
- package/src/tooling/node-browser/execution.d.ts +40 -0
- package/src/tooling/node-browser/execution.js +308 -0
- package/src/tooling/node-browser/static-server.js +189 -0
- package/src/tooling/node-browser/transport.d.ts +80 -0
- package/src/tooling/node-browser/transport.js +622 -0
- package/src/tooling/node-browser-command-runner.d.ts +18 -78
- package/src/tooling/node-browser-command-runner.js +10 -1112
- package/src/tooling/node-capsule-artifact-store.js +44 -0
- package/src/tooling/node-command-execution.js +237 -0
- package/src/tooling/node-command-runner.js +26 -205
- package/src/tooling/node-converter/input.js +521 -0
- package/src/tooling/node-converter/output.js +244 -0
- package/src/tooling/node-converter-chunk-batches.js +21 -0
- package/src/tooling/node-converter.js +7 -720
- package/src/tooling/node-source-runtime.js +16 -276
- package/src/tooling/node-webgpu-provider.v1.json +33 -0
- package/src/tooling/node-webgpu.js +178 -453
- package/src/tooling/opfs-cache.d.ts +5 -0
- package/src/tooling/opfs-cache.js +4 -0
- package/src/tooling/production-release.d.ts +19 -0
- package/src/tooling/production-release.js +647 -0
- package/src/tooling/program-bundle/embedding-reference.js +19 -0
- package/src/tooling/program-bundle/materialize.js +324 -0
- package/src/tooling/program-bundle/rerank-reference.js +22 -0
- package/src/tooling/program-bundle/sequence-reference.js +42 -0
- package/src/tooling/program-bundle/validation.js +31 -0
- package/src/tooling/program-bundle/wgsl-closure.js +254 -0
- package/src/tooling/program-bundle-parity.d.ts +11 -4
- package/src/tooling/program-bundle-parity.js +262 -109
- package/src/tooling/program-bundle.d.ts +7 -0
- package/src/tooling/program-bundle.js +186 -591
- package/src/tooling/provider-v1-contract.d.ts +61 -0
- package/src/tooling/provider-v1-contract.js +192 -0
- package/src/tooling/registered-variant-calibration-job.d.ts +98 -0
- package/src/tooling/registered-variant-calibration-job.js +210 -0
- package/src/tooling/registered-variant-calibration.d.ts +167 -7
- package/src/tooling/registered-variant-calibration.js +57 -7
- package/src/tooling/runtime-optimization/candidates.js +664 -0
- package/src/tooling/runtime-optimization-campaign.js +166 -0
- package/src/tooling/runtime-optimization-index.js +16 -1
- package/src/tooling/runtime-optimization.d.ts +44 -0
- package/src/tooling/runtime-optimization.js +13 -656
- package/src/tooling/source-intake.d.ts +1 -1
- package/src/tooling/source-intake.js +34 -17
- package/src/tooling/source-package-profiles.js +1 -1
- package/src/tooling/source-runtime/bundle-materialize.js +298 -0
- package/src/tooling/source-runtime/bundle-plan.js +121 -0
- package/src/tooling/source-runtime/node-loader.js +258 -0
- package/src/tooling/source-runtime-bundle.js +26 -411
- package/src/tooling/source-runtime-converter-config.js +1 -1
- package/src/tooling-experimental-exports.js +0 -5
- package/src/tooling-exports/evidence.d.ts +14 -3
- package/src/tooling-exports/evidence.js +14 -3
- package/src/tooling-exports/storage.d.ts +7 -0
- package/src/tooling-exports/storage.js +5 -0
- package/src/tooling-exports.d.ts +3 -0
- package/src/tooling-exports.js +2 -0
- package/src/tooling-exports.shared.d.ts +17 -3
- package/src/tooling-exports.shared.js +9 -3
- package/src/training.d.ts +6 -6
- package/src/training.js +4 -4
- package/src/version.js +1 -1
- package/tests/kernels/browser/kernel-suite.js +512 -0
- package/tests/kernels/browser/test-page.js +170 -14
- package/tests/kernels/harness/tolerance.js +1 -0
- package/tests/kernels/reference/index.js +13 -1
- package/tests/kernels/reference/vision.js +160 -0
- package/src/converter/quantizer.d.ts +0 -121
- package/src/gpu/kernels/backward/utils.d.ts +0 -90
- package/src/gpu/kernels/uniform-utils.d.ts +0 -66
- package/src/gpu/kernels/utils.d.ts +0 -147
- package/src/inference/pipelines/text/debug-utils/utils.js +0 -118
- package/src/inference/pipelines/text/logits/utils.d.ts +0 -58
- package/src/inference/pipelines/text/moe-cpu-gptoss.js +0 -115
- package/src/inference/pipelines/text/moe-helpers.js +0 -21
- package/src/rules/converter/execution.rules.json +0 -20
- package/src/rules/converter/tensor-roles.rules.json +0 -13
- package/src/rules/converter/tokenizer.rules.json +0 -12
- package/src/rules/inference/attention.rules.json +0 -69
- package/src/rules/inference/config.rules.json +0 -82
- package/src/rules/inference/dtype.rules.json +0 -113
- package/src/rules/inference/execution.rules.json +0 -195
- package/src/rules/inference/ffn.rules.json +0 -123
- package/src/rules/inference/layer-pattern.rules.json +0 -16
- package/src/rules/inference/layer.rules.json +0 -7
- package/src/rules/inference/moe.rules.json +0 -145
- package/src/rules/kernels/attention.rules.json +0 -68
- package/src/rules/kernels/conv2d.rules.json +0 -6
- package/src/rules/kernels/depthwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/dequant.rules.json +0 -58
- package/src/rules/kernels/energy.rules.json +0 -22
- package/src/rules/kernels/fused-ffn.rules.json +0 -15
- package/src/rules/kernels/fused-matmul-residual.rules.json +0 -6
- package/src/rules/kernels/fused-matmul-rmsnorm.rules.json +0 -8
- package/src/rules/kernels/gather.rules.json +0 -93
- package/src/rules/kernels/gelu.rules.json +0 -11
- package/src/rules/kernels/grouped-pointwise-conv2d.rules.json +0 -6
- package/src/rules/kernels/groupnorm.rules.json +0 -10
- package/src/rules/kernels/kv_quantize.rules.json +0 -9
- package/src/rules/kernels/layernorm.rules.json +0 -6
- package/src/rules/kernels/linear-attention.rules.json +0 -6
- package/src/rules/kernels/lm-head-argmax.rules.json +0 -10
- package/src/rules/kernels/matmul.rules.json +0 -95
- package/src/rules/kernels/modulate.rules.json +0 -6
- package/src/rules/kernels/moe.rules.gptoss.json +0 -105
- package/src/rules/kernels/moe.rules.json +0 -14
- package/src/rules/kernels/moe.rules.mixtral.json +0 -75
- package/src/rules/kernels/pixel_shuffle.rules.json +0 -6
- package/src/rules/kernels/relu.rules.json +0 -6
- package/src/rules/kernels/rep-penalty.rules.json +0 -6
- package/src/rules/kernels/repeat-channels.rules.json +0 -6
- package/src/rules/kernels/residual.rules.json +0 -12
- package/src/rules/kernels/rmsnorm-qk.rules.json +0 -6
- package/src/rules/kernels/rmsnorm.rules.json +0 -46
- package/src/rules/kernels/rope-qk.rules.json +0 -6
- package/src/rules/kernels/rope.rules.json +0 -6
- package/src/rules/kernels/sample.rules.json +0 -6
- package/src/rules/kernels/scale.rules.json +0 -8
- package/src/rules/kernels/silu.rules.json +0 -22
- package/src/rules/kernels/softmax.rules.json +0 -25
- package/src/rules/kernels/split-qg.rules.json +0 -6
- package/src/rules/kernels/split-qkv.rules.json +0 -6
- package/src/rules/kernels/upsample2d.rules.json +0 -6
- package/src/rules/loader/tensor-loader.rules.json +0 -42
- package/src/rules/loader/weights.rules.json +0 -42
- package/src/rules/tooling/command-runtime.rules.json +0 -65
- package/src/tooling/hf-registry-utils.js +0 -395
- /package/src/converter/{tokenizer-utils.js → tokenizer-special-tokens.js} +0 -0
- /package/src/{inference/pipelines/text/bdpa-steamroller.js → experimental/bdpa/steamroller.js} +0 -0
- /package/src/{inference/pipelines → experimental}/diffusion/scheduler.js +0 -0
- /package/src/{inference/pipelines → experimental}/energy/pipeline.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.d.ts +0 -0
- /package/src/{inference/pipelines → experimental}/energy/quintel.js +0 -0
- /package/src/{utils → formats}/array-buffer.js +0 -0
- /package/src/{utils → formats}/canonical-hash.js +0 -0
- /package/src/{utils → formats}/clone-json.js +0 -0
- /package/src/{utils → formats}/encode-utf8.js +0 -0
- /package/src/{utils → formats}/load-json.js +0 -0
- /package/src/{utils → formats}/plain-object.js +0 -0
- /package/src/{utils → formats}/sha256.js +0 -0
- /package/src/{utils → formats}/stable-sort-object.js +0 -0
- /package/src/{gpu/kernels → rules}/rule-matcher.js +0 -0
- /package/src/{utils → storage}/hf-resolve-url.js +0 -0
- /package/src/{utils → storage}/runtime-env.js +0 -0
- /package/src/tooling/{command-runner-shared.js → command-runtime-execution.js} +0 -0
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
1
|
import { getMemoryCapabilities } from '../memory/capability.js';
|
|
4
2
|
import { detectUnifiedMemory } from '../memory/unified-detect.js';
|
|
5
3
|
import { getHeapManager } from '../memory/heap-manager.js';
|
|
@@ -23,14 +21,11 @@ import {
|
|
|
23
21
|
import { getExpertCache } from './experts/expert-cache.js';
|
|
24
22
|
import { formatBytes } from '../storage/quota.js';
|
|
25
23
|
import { log, trace as debugTrace } from '../debug/index.js';
|
|
26
|
-
import { isWeightBuffer } from '../gpu/weight-buffer.js';
|
|
27
|
-
|
|
28
|
-
|
|
24
|
+
import { isGpuBufferInstance, isWeightBuffer } from '../gpu/weight-buffer.js';
|
|
29
25
|
import { createShardCache } from './shard-cache.js';
|
|
30
26
|
import { validateManifestInference } from '../config/schema/index.js';
|
|
31
27
|
import { getRuntimeConfig } from '../config/runtime.js';
|
|
32
|
-
|
|
33
|
-
// Import helper modules for refactored logic
|
|
28
|
+
import { loadGpuTensor as loadGpuTensorImpl } from './gpu-tensor-materialization.js';
|
|
34
29
|
import { buildTensorLocations } from './shard-resolver.js';
|
|
35
30
|
import {
|
|
36
31
|
needsNormWeightOffset,
|
|
@@ -44,12 +39,13 @@ import {
|
|
|
44
39
|
loadTensorToCPU,
|
|
45
40
|
isLiteRTAffineInt4FusedEligible,
|
|
46
41
|
} from './tensors/tensor-loader.js';
|
|
47
|
-
import {
|
|
42
|
+
import { detectMoE } from './model-load-validation.js';
|
|
43
|
+
import { getTensorShardIndices } from './tensor-shard-indices.js';
|
|
48
44
|
import { loadEmbeddings } from './embedding-loader.js';
|
|
49
45
|
import { loadPerLayerInputWeights } from './per-layer-input-loader.js';
|
|
50
46
|
import { loadLayer } from './layer-loader.js';
|
|
51
47
|
import { loadFinalWeights } from './final-weights-loader.js';
|
|
52
|
-
import { cloneJsonValue } from '../
|
|
48
|
+
import { cloneJsonValue } from '../formats/clone-json.js';
|
|
53
49
|
import { assertFunctionalDescriptorManifest } from '../formats/rdrr/functional-descriptor.js';
|
|
54
50
|
import {
|
|
55
51
|
loadExpert as loadExpertFromModule,
|
|
@@ -58,6 +54,20 @@ import {
|
|
|
58
54
|
} from './experts/expert-loader.js';
|
|
59
55
|
import { assembleShardData } from './tensors/tensor-reader.js';
|
|
60
56
|
import { hasSourceTransform } from './tensors/source-transform.js';
|
|
57
|
+
import {
|
|
58
|
+
load as loadImpl,
|
|
59
|
+
_loadTensor as _loadTensorImpl,
|
|
60
|
+
_getDescriptorShardFiles as _getDescriptorShardFilesImpl,
|
|
61
|
+
_loadDescriptorShardFile as _loadDescriptorShardFileImpl,
|
|
62
|
+
_assertDescriptorHash as _assertDescriptorHashImpl,
|
|
63
|
+
_assembleFunctionalDescriptorData as _assembleFunctionalDescriptorDataImpl,
|
|
64
|
+
_assembleShardData as _assembleShardDataImpl,
|
|
65
|
+
_shouldStreamUploadToGPU as _shouldStreamUploadToGPUImpl,
|
|
66
|
+
_assembleShardDataToGpuBuffer as _assembleShardDataToGpuBufferImpl,
|
|
67
|
+
_loadEmbeddings as _loadEmbeddingsImpl,
|
|
68
|
+
_loadLayer as _loadLayerImpl,
|
|
69
|
+
_loadFinalWeights as _loadFinalWeightsImpl,
|
|
70
|
+
} from './model-executor.js';
|
|
61
71
|
|
|
62
72
|
let loraModulePromise = null;
|
|
63
73
|
|
|
@@ -66,11 +76,6 @@ async function getExperimentalLoRAModule() {
|
|
|
66
76
|
return loraModulePromise;
|
|
67
77
|
}
|
|
68
78
|
|
|
69
|
-
function hasExpertGroups(manifest) {
|
|
70
|
-
if (!manifest?.groups) return false;
|
|
71
|
-
return Object.keys(manifest.groups).some((groupId) => groupId.includes('.expert.'));
|
|
72
|
-
}
|
|
73
|
-
|
|
74
79
|
function isDirectSourceRuntimeManifest(manifest) {
|
|
75
80
|
const sourceRuntime = manifest?.metadata?.sourceRuntime;
|
|
76
81
|
const mode = typeof sourceRuntime?.mode === 'string'
|
|
@@ -82,110 +87,6 @@ function isDirectSourceRuntimeManifest(manifest) {
|
|
|
82
87
|
return mode === 'direct-source' && sourceKind !== 'rdrr';
|
|
83
88
|
}
|
|
84
89
|
|
|
85
|
-
/**
|
|
86
|
-
* Detect whether a manifest describes a Mixture-of-Experts model.
|
|
87
|
-
* Validates that manifests with expert groups also have moeConfig.
|
|
88
|
-
* Used by both setManifest() and load() to avoid duplicating MoE detection logic.
|
|
89
|
-
*/
|
|
90
|
-
function detectMoE(manifest) {
|
|
91
|
-
const moeConfig = manifest.moeConfig;
|
|
92
|
-
const isMoE = moeConfig != null && (moeConfig.numExperts ?? 0) > 1;
|
|
93
|
-
if (!isMoE && hasExpertGroups(manifest)) {
|
|
94
|
-
throw new Error(
|
|
95
|
-
`Manifest "${manifest.modelId ?? 'unknown'}" missing moeConfig for MoE model. Re-convert with moeConfig.`
|
|
96
|
-
);
|
|
97
|
-
}
|
|
98
|
-
return isMoE;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
function isGpuBufferInstance(value) {
|
|
102
|
-
return typeof GPUBuffer !== 'undefined' && value instanceof GPUBuffer;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
function nowMs() {
|
|
106
|
-
return typeof performance !== 'undefined' && typeof performance.now === 'function'
|
|
107
|
-
? performance.now()
|
|
108
|
-
: Date.now();
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
function roundLoadTimingMs(value) {
|
|
112
|
-
return Number.isFinite(value) ? Number(value.toFixed(3)) : null;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
function createLoadTiming(modelId, hasCustomLoader) {
|
|
116
|
-
return {
|
|
117
|
-
schemaVersion: 1,
|
|
118
|
-
source: 'doppler-loader',
|
|
119
|
-
modelId: typeof modelId === 'string' ? modelId : null,
|
|
120
|
-
status: 'running',
|
|
121
|
-
customShardLoader: hasCustomLoader === true,
|
|
122
|
-
byteAccountingMode: hasCustomLoader === true
|
|
123
|
-
? 'custom-loader-read-progress'
|
|
124
|
-
: 'full-shard-progress',
|
|
125
|
-
totalBytes: null,
|
|
126
|
-
totalShards: null,
|
|
127
|
-
bytesLoaded: 0,
|
|
128
|
-
shardsLoaded: 0,
|
|
129
|
-
bytesPerSecond: null,
|
|
130
|
-
phasesMs: {
|
|
131
|
-
preflight: null,
|
|
132
|
-
tensorLocations: null,
|
|
133
|
-
embeddings: null,
|
|
134
|
-
layers: null,
|
|
135
|
-
finalWeights: null,
|
|
136
|
-
cleanup: null,
|
|
137
|
-
},
|
|
138
|
-
layers: {
|
|
139
|
-
count: null,
|
|
140
|
-
totalMs: null,
|
|
141
|
-
meanMs: null,
|
|
142
|
-
maxMs: null,
|
|
143
|
-
maxLayer: null,
|
|
144
|
-
},
|
|
145
|
-
totalMs: null,
|
|
146
|
-
failedPhase: null,
|
|
147
|
-
error: null,
|
|
148
|
-
};
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
function finishLoadPhase(loadTiming, phase, startMs) {
|
|
152
|
-
if (!loadTiming?.phasesMs || !phase) return;
|
|
153
|
-
loadTiming.phasesMs[phase] = roundLoadTimingMs(nowMs() - startMs);
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
function finishLoadTiming(loadTiming, status, startMs, error = null, failedPhase = null) {
|
|
157
|
-
if (!loadTiming) return;
|
|
158
|
-
const totalMs = nowMs() - startMs;
|
|
159
|
-
loadTiming.status = status;
|
|
160
|
-
loadTiming.totalMs = roundLoadTimingMs(totalMs);
|
|
161
|
-
loadTiming.bytesPerSecond = totalMs > 0 && Number.isFinite(loadTiming.bytesLoaded)
|
|
162
|
-
? Math.round((loadTiming.bytesLoaded / totalMs) * 1000)
|
|
163
|
-
: null;
|
|
164
|
-
if (status === 'failed') {
|
|
165
|
-
loadTiming.failedPhase = failedPhase;
|
|
166
|
-
loadTiming.error = error?.message ?? String(error ?? 'unknown load error');
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
function getTensorShardIndices(location) {
|
|
171
|
-
if (Array.isArray(location?.spans) && location.spans.length > 0) {
|
|
172
|
-
return Array.from(new Set(location.spans.map((span) => span?.shardIndex).filter(Number.isInteger)));
|
|
173
|
-
}
|
|
174
|
-
return Number.isInteger(location?.shardIndex) ? [location.shardIndex] : [];
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
function annotateTensorLoadError(error, name, location, details = {}) {
|
|
178
|
-
return annotateWeightLoadError(error, {
|
|
179
|
-
tensorName: name,
|
|
180
|
-
tensorRole: location?.role ?? null,
|
|
181
|
-
tensorDtype: location?.dtype ?? null,
|
|
182
|
-
tensorShape: Array.isArray(location?.shape) ? [...location.shape] : null,
|
|
183
|
-
tensorSizeBytes: Number.isFinite(location?.size) ? location.size : null,
|
|
184
|
-
tensorShardIndices: getTensorShardIndices(location),
|
|
185
|
-
...details,
|
|
186
|
-
});
|
|
187
|
-
}
|
|
188
|
-
|
|
189
90
|
// Re-export types for backward compatibility
|
|
190
91
|
export {
|
|
191
92
|
// Types are in .d.ts file
|
|
@@ -195,7 +96,6 @@ export {
|
|
|
195
96
|
// DopplerLoader Class
|
|
196
97
|
// ============================================================================
|
|
197
98
|
|
|
198
|
-
|
|
199
99
|
export class DopplerLoader {
|
|
200
100
|
// Capabilities
|
|
201
101
|
|
|
@@ -260,11 +160,11 @@ export class DopplerLoader {
|
|
|
260
160
|
|
|
261
161
|
// Loading configuration
|
|
262
162
|
|
|
263
|
-
|
|
264
|
-
|
|
163
|
+
_loadingConfig;
|
|
164
|
+
_loaderDebug = null;
|
|
265
165
|
|
|
266
|
-
|
|
267
|
-
|
|
166
|
+
_perLayerInputSession = null;
|
|
167
|
+
_loadAuxiliaryFile = null;
|
|
268
168
|
|
|
269
169
|
// Fused Q4_K matmul: skip dequantization for matmul weights, use fused kernel
|
|
270
170
|
|
|
@@ -275,41 +175,40 @@ export class DopplerLoader {
|
|
|
275
175
|
q4kLayout = null;
|
|
276
176
|
|
|
277
177
|
keepF32Weights = false;
|
|
178
|
+
keepBF16Weights = false;
|
|
278
179
|
|
|
279
180
|
q4kMaterializationMode = 'dense';
|
|
280
|
-
|
|
281
181
|
q4kFusedRoles = [];
|
|
282
182
|
|
|
283
183
|
// Internal tracking
|
|
284
184
|
|
|
285
|
-
|
|
185
|
+
_normOffsetLogged = false;
|
|
286
186
|
|
|
287
|
-
|
|
187
|
+
_normOffsetDebugLogged = false;
|
|
288
188
|
|
|
289
|
-
|
|
189
|
+
_memoryMonitor = null;
|
|
290
190
|
|
|
291
|
-
|
|
292
|
-
|
|
191
|
+
_tensorsJsonUrl = null;
|
|
192
|
+
_loadTensorsJson = null;
|
|
293
193
|
|
|
294
|
-
|
|
194
|
+
_loadShardOverride = null;
|
|
295
195
|
|
|
296
|
-
|
|
196
|
+
_layerShardMap = new Map();
|
|
297
197
|
|
|
298
|
-
|
|
299
198
|
constructor(loadingConfig) {
|
|
300
199
|
if (!loadingConfig) {
|
|
301
200
|
log.debug('Loader', 'No explicit loadingConfig provided; falling back to getRuntimeConfig().loading');
|
|
302
201
|
}
|
|
303
|
-
this
|
|
202
|
+
this._loadingConfig = loadingConfig ?? getRuntimeConfig().loading;
|
|
304
203
|
this.shardCache = createShardCache(
|
|
305
|
-
this
|
|
306
|
-
this
|
|
204
|
+
this._loadingConfig.shardCache.opfsEntries,
|
|
205
|
+
this._loadingConfig.shardCache
|
|
307
206
|
);
|
|
308
207
|
}
|
|
309
208
|
|
|
310
209
|
|
|
311
210
|
setLoadingConfig(config) {
|
|
312
|
-
this
|
|
211
|
+
this._loadingConfig = config;
|
|
313
212
|
this.shardCache.configure({
|
|
314
213
|
loadingConfig: config.shardCache,
|
|
315
214
|
maxEntries: config.shardCache.opfsEntries,
|
|
@@ -323,11 +222,11 @@ export class DopplerLoader {
|
|
|
323
222
|
}
|
|
324
223
|
|
|
325
224
|
setLoaderDebugConfig(loaderDebug) {
|
|
326
|
-
this
|
|
225
|
+
this._loaderDebug = loaderDebug ?? null;
|
|
327
226
|
}
|
|
328
227
|
|
|
329
228
|
setPerLayerInputSession(sessionConfig) {
|
|
330
|
-
this
|
|
229
|
+
this._perLayerInputSession = sessionConfig ?? null;
|
|
331
230
|
}
|
|
332
231
|
|
|
333
232
|
|
|
@@ -335,6 +234,7 @@ export class DopplerLoader {
|
|
|
335
234
|
this.useFusedQ4K = config.useFusedQ4K;
|
|
336
235
|
this.q4kLayout = config.q4kLayout;
|
|
337
236
|
this.keepF32Weights = config.keepF32Weights;
|
|
237
|
+
this.keepBF16Weights = config.keepBF16Weights === true;
|
|
338
238
|
this.q4kMaterializationMode = config.q4kMaterializationMode ?? 'dense';
|
|
339
239
|
this.q4kFusedRoles = Array.isArray(config.q4kFusedRoles)
|
|
340
240
|
? [...new Set(config.q4kFusedRoles.map((role) => String(role)).filter(Boolean))].sort()
|
|
@@ -342,7 +242,7 @@ export class DopplerLoader {
|
|
|
342
242
|
}
|
|
343
243
|
|
|
344
244
|
|
|
345
|
-
|
|
245
|
+
_getMemoryState() {
|
|
346
246
|
return {
|
|
347
247
|
shardCacheBytes: this.shardCache.totalBytes,
|
|
348
248
|
shardCount: this.shardCache.size,
|
|
@@ -351,14 +251,14 @@ export class DopplerLoader {
|
|
|
351
251
|
};
|
|
352
252
|
}
|
|
353
253
|
|
|
354
|
-
|
|
355
|
-
const logIntervalMs = this
|
|
356
|
-
this
|
|
357
|
-
this
|
|
254
|
+
_startMemoryLogging() {
|
|
255
|
+
const logIntervalMs = this._loadingConfig.memoryManagement.logIntervalMs;
|
|
256
|
+
this._memoryMonitor = new MemoryMonitor(logIntervalMs);
|
|
257
|
+
this._memoryMonitor.start(() => this._getMemoryState());
|
|
358
258
|
}
|
|
359
259
|
|
|
360
|
-
|
|
361
|
-
const budgetConfig = this
|
|
260
|
+
_assertResidentBudget(phase) {
|
|
261
|
+
const budgetConfig = this._loadingConfig?.memoryManagement?.budget;
|
|
362
262
|
if (!budgetConfig || budgetConfig.enabled !== true) {
|
|
363
263
|
return;
|
|
364
264
|
}
|
|
@@ -381,10 +281,10 @@ export class DopplerLoader {
|
|
|
381
281
|
}
|
|
382
282
|
|
|
383
283
|
|
|
384
|
-
|
|
385
|
-
if (this
|
|
386
|
-
this
|
|
387
|
-
this
|
|
284
|
+
_stopMemoryLogging(phase = 'complete') {
|
|
285
|
+
if (this._memoryMonitor) {
|
|
286
|
+
this._memoryMonitor.stop(phase, () => this._getMemoryState());
|
|
287
|
+
this._memoryMonitor = null;
|
|
388
288
|
}
|
|
389
289
|
}
|
|
390
290
|
|
|
@@ -394,32 +294,32 @@ export class DopplerLoader {
|
|
|
394
294
|
loadRange: options.loadShardRange ?? null,
|
|
395
295
|
streamRange: options.streamShardRange ?? null,
|
|
396
296
|
});
|
|
397
|
-
this
|
|
297
|
+
this._loadAuxiliaryFile = typeof options.loadAuxiliaryFile === 'function'
|
|
398
298
|
? options.loadAuxiliaryFile
|
|
399
299
|
: null;
|
|
400
300
|
}
|
|
401
301
|
|
|
402
302
|
setAuxiliaryFileLoader(loadAuxiliaryFile) {
|
|
403
|
-
this
|
|
303
|
+
this._loadAuxiliaryFile = typeof loadAuxiliaryFile === 'function' ? loadAuxiliaryFile : null;
|
|
404
304
|
}
|
|
405
305
|
|
|
406
306
|
|
|
407
307
|
setTensorsJsonUrl(url) {
|
|
408
|
-
this
|
|
308
|
+
this._tensorsJsonUrl = url;
|
|
409
309
|
}
|
|
410
310
|
|
|
411
311
|
setTensorsJsonLoader(loadTensorsJson) {
|
|
412
|
-
this
|
|
312
|
+
this._loadTensorsJson = typeof loadTensorsJson === 'function' ? loadTensorsJson : null;
|
|
413
313
|
}
|
|
414
314
|
|
|
415
315
|
|
|
416
|
-
async
|
|
316
|
+
async _loadShard(shardIndex, options) {
|
|
417
317
|
return this.shardCache.load(shardIndex, options);
|
|
418
318
|
}
|
|
419
319
|
|
|
420
320
|
|
|
421
|
-
|
|
422
|
-
return this
|
|
321
|
+
_getLoadShard() {
|
|
322
|
+
return this._loadShardOverride ?? ((idx, options) => this._loadShard(idx, options));
|
|
423
323
|
}
|
|
424
324
|
|
|
425
325
|
|
|
@@ -468,18 +368,18 @@ export class DopplerLoader {
|
|
|
468
368
|
async loadLoRAWeights(manifest) {
|
|
469
369
|
const prevManifest = this.manifest;
|
|
470
370
|
const prevLocations = new Map(this.tensorLocations);
|
|
471
|
-
const prevLayerShardMap = new Map(this
|
|
371
|
+
const prevLayerShardMap = new Map(this._layerShardMap);
|
|
472
372
|
|
|
473
373
|
try {
|
|
474
374
|
const { loadLoRAWeights } = await getExperimentalLoRAModule();
|
|
475
375
|
this.manifest = manifest;
|
|
476
376
|
// We must rebuild locations so _loadTensor finds them
|
|
477
|
-
await this
|
|
478
|
-
this
|
|
377
|
+
await this._buildTensorLocations();
|
|
378
|
+
this._logWeightBreakdown();
|
|
479
379
|
|
|
480
380
|
return await loadLoRAWeights(
|
|
481
381
|
manifest,
|
|
482
|
-
(name, toGPU, silent) => this
|
|
382
|
+
(name, toGPU, silent) => this._loadTensor(name, toGPU, silent)
|
|
483
383
|
);
|
|
484
384
|
} finally {
|
|
485
385
|
// Always restore previous state, even if buildTensorLocations or the
|
|
@@ -487,377 +387,37 @@ export class DopplerLoader {
|
|
|
487
387
|
// inconsistent intermediate state.
|
|
488
388
|
this.manifest = prevManifest;
|
|
489
389
|
this.tensorLocations = prevLocations;
|
|
490
|
-
this
|
|
390
|
+
this._layerShardMap = prevLayerShardMap;
|
|
491
391
|
}
|
|
492
392
|
}
|
|
493
393
|
|
|
494
394
|
|
|
495
|
-
|
|
395
|
+
_resolveWeightLayout(location) {
|
|
496
396
|
return resolveWeightLayout(location);
|
|
497
397
|
}
|
|
498
398
|
|
|
499
399
|
|
|
500
|
-
|
|
400
|
+
_shouldStreamLargeWeight(name, location, label) {
|
|
501
401
|
const manifestOverrides = this.manifest?.inference?.largeWeights?.gpuResidentOverrides ?? null;
|
|
502
402
|
return shouldStreamLargeWeight(name, location, label, this.gpuCapabilities, this.keepF32Weights, manifestOverrides);
|
|
503
403
|
}
|
|
504
404
|
|
|
505
405
|
|
|
506
406
|
async load(modelId, options = {}) {
|
|
507
|
-
|
|
508
|
-
if (verifyHashes == null) {
|
|
509
|
-
throw new Error('Loader.load requires explicit verifyHashes (runtime.loading.shardCache.verifyHashes).');
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
if (!this.heapManager) {
|
|
513
|
-
await this.init();
|
|
514
|
-
}
|
|
515
|
-
|
|
516
|
-
// Check order matters: isLoaded is the fast-path indicator; modelId catches
|
|
517
|
-
// partial loads that set the ID before completing; tensorLocations/shardCache
|
|
518
|
-
// detect interrupted builds; layers/experts/gpuBuffers catch residual GPU
|
|
519
|
-
// state from a prior model that was never fully unloaded.
|
|
520
|
-
const hasExistingModelState =
|
|
521
|
-
this.isLoaded ||
|
|
522
|
-
this.modelId !== null ||
|
|
523
|
-
this.tensorLocations.size > 0 ||
|
|
524
|
-
this.shardCache.size > 0 ||
|
|
525
|
-
this.layers.size > 0 ||
|
|
526
|
-
this.experts.size > 0 ||
|
|
527
|
-
this.gpuBuffers.size > 0;
|
|
528
|
-
|
|
529
|
-
const preservedManifest = this.shardCache.hasCustomLoader ? this.manifest : null;
|
|
530
|
-
|
|
531
|
-
if (hasExistingModelState) {
|
|
532
|
-
await this.unload();
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
if (preservedManifest) {
|
|
536
|
-
this.manifest = preservedManifest;
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
log.info('Loader', `Loading: ${modelId}`);
|
|
540
|
-
this.modelId = modelId;
|
|
541
|
-
const loadTimingStart = nowMs();
|
|
542
|
-
let activeLoadPhase = 'preflight';
|
|
543
|
-
let phaseStart = loadTimingStart;
|
|
544
|
-
this.loadTiming = createLoadTiming(modelId, this.shardCache.hasCustomLoader);
|
|
545
|
-
|
|
546
|
-
this.#startMemoryLogging();
|
|
547
|
-
this.#assertResidentBudget('load start');
|
|
548
|
-
|
|
549
|
-
if (!this.shardCache.hasCustomLoader) {
|
|
550
|
-
await openModelStore(modelId);
|
|
551
|
-
const manifestJson = await loadManifestFromStore();
|
|
552
|
-
this.manifest = parseManifest(manifestJson);
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
if (!this.manifest) {
|
|
556
|
-
throw new Error('No manifest available. Set manifest via setManifest() or ensure OPFS has the model.');
|
|
557
|
-
}
|
|
558
|
-
|
|
559
|
-
validateManifestInference(this.manifest);
|
|
560
|
-
|
|
561
|
-
this.isMoE = detectMoE(this.manifest);
|
|
562
|
-
|
|
563
|
-
this.shardCache.configureForModel(this.manifest, this.shardCache.hasCustomLoader);
|
|
564
|
-
|
|
565
|
-
if (!this.isMoE && !this.isUnifiedMemory) {
|
|
566
|
-
log.warn('Loader', 'Dense model on discrete GPU - performance limited. Consider MoE model.');
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
if (!this.shardCache.hasCustomLoader) {
|
|
570
|
-
const integrity = await verifyIntegrity({ checkHashes: false });
|
|
571
|
-
if (!integrity.valid) {
|
|
572
|
-
throw new Error(
|
|
573
|
-
`Artifact contract preflight failed for "${this.manifest?.modelId ?? modelId}". ` +
|
|
574
|
-
`Missing shards: ${integrity.missingShards.length}, ` +
|
|
575
|
-
`corrupt shards: ${integrity.corruptShards.length}. ` +
|
|
576
|
-
'Re-import, re-download, or provide a manifest with a valid weightsRef.'
|
|
577
|
-
);
|
|
578
|
-
}
|
|
579
|
-
}
|
|
580
|
-
|
|
581
|
-
const totalBytes = (this.manifest.shards || []).reduce((sum, s) => sum + (s.size || 0), 0);
|
|
582
|
-
const totalShards = this.manifest.shards?.length || 0;
|
|
583
|
-
this.loadTiming.totalBytes = totalBytes;
|
|
584
|
-
this.loadTiming.totalShards = totalShards;
|
|
585
|
-
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
586
|
-
|
|
587
|
-
activeLoadPhase = 'tensorLocations';
|
|
588
|
-
phaseStart = nowMs();
|
|
589
|
-
await this.#buildTensorLocations();
|
|
590
|
-
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
591
|
-
|
|
592
|
-
const loadStartTime = Date.now();
|
|
593
|
-
let bytesLoaded = 0;
|
|
594
|
-
let shardsLoaded = 0;
|
|
595
|
-
this.shardCache.resetCustomReadStats();
|
|
596
|
-
|
|
597
|
-
const syncCustomReadStats = () => {
|
|
598
|
-
if (!this.shardCache.hasCustomLoader) return;
|
|
599
|
-
const stats = this.shardCache.customReadStats;
|
|
600
|
-
bytesLoaded = stats.bytesRead;
|
|
601
|
-
shardsLoaded = stats.shardsRead;
|
|
602
|
-
this.loadTiming.bytesLoaded = bytesLoaded;
|
|
603
|
-
this.loadTiming.shardsLoaded = shardsLoaded;
|
|
604
|
-
};
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
const reportProgress = (stage, baseProgress, detail) => {
|
|
608
|
-
if (!onProgress || typeof onProgress !== 'function') return;
|
|
609
|
-
syncCustomReadStats();
|
|
610
|
-
const elapsed = (Date.now() - loadStartTime) / 1000;
|
|
611
|
-
const speed = elapsed > 0 ? bytesLoaded / elapsed : 0;
|
|
612
|
-
const speedStr = speed > 0 ? `${formatBytes(speed)}/s` : '';
|
|
613
|
-
const message = detail ||
|
|
614
|
-
`${formatBytes(bytesLoaded)} / ${formatBytes(totalBytes)} ${speedStr ? `- ${speedStr}` : ''}`;
|
|
615
|
-
onProgress({
|
|
616
|
-
stage,
|
|
617
|
-
progress: baseProgress,
|
|
618
|
-
shard: shardsLoaded,
|
|
619
|
-
totalShards,
|
|
620
|
-
bytesLoaded,
|
|
621
|
-
totalBytes,
|
|
622
|
-
bytesPerSecond: speed,
|
|
623
|
-
message,
|
|
624
|
-
});
|
|
625
|
-
};
|
|
626
|
-
|
|
627
|
-
if (onProgress) {
|
|
628
|
-
onProgress({ stage: 'manifest', progress: 0.05, message: 'Parsing manifest...' });
|
|
629
|
-
}
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
const loadedShardIndices = new Set();
|
|
633
|
-
let inLayerPhase = false;
|
|
634
|
-
const originalLoadShard = (shardIndex, options) => this.#loadShard(shardIndex, options);
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
this.#loadShardOverride = async (shardIndex, options) => {
|
|
638
|
-
const shardInfo = this.manifest?.shards?.[shardIndex];
|
|
639
|
-
const shardSize = shardInfo?.size || 0;
|
|
640
|
-
const shardName = shardInfo?.filename ?? `index=${shardIndex}`;
|
|
641
|
-
let data;
|
|
642
|
-
try {
|
|
643
|
-
data = await originalLoadShard(shardIndex, options);
|
|
644
|
-
} catch (error) {
|
|
645
|
-
const modelId = this.manifest?.modelId ?? 'unknown';
|
|
646
|
-
const shardUrl = shardInfo?.url ?? shardInfo?.path ?? 'unknown';
|
|
647
|
-
const sizeStr = shardSize > 0 ? `, size=${formatBytes(shardSize)}` : '';
|
|
648
|
-
log.error(
|
|
649
|
-
'Loader',
|
|
650
|
-
`Failed to load shard ${shardIndex}/${totalShards} "${shardName}" ` +
|
|
651
|
-
`for model "${modelId}" (url=${shardUrl}${sizeStr}): ${error.message}`
|
|
652
|
-
);
|
|
653
|
-
throw error;
|
|
654
|
-
}
|
|
655
|
-
|
|
656
|
-
if (!loadedShardIndices.has(shardIndex)) {
|
|
657
|
-
loadedShardIndices.add(shardIndex);
|
|
658
|
-
bytesLoaded += shardSize;
|
|
659
|
-
shardsLoaded++;
|
|
660
|
-
this.loadTiming.bytesLoaded = bytesLoaded;
|
|
661
|
-
this.loadTiming.shardsLoaded = shardsLoaded;
|
|
662
|
-
if (!inLayerPhase) {
|
|
663
|
-
const pct = 0.1 + Math.min(bytesLoaded / totalBytes, 1.0) * 0.7;
|
|
664
|
-
const elapsed = (Date.now() - loadStartTime) / 1000;
|
|
665
|
-
const speed = elapsed > 0 ? bytesLoaded / elapsed : 0;
|
|
666
|
-
const sourceInfo = this.shardCache.lastSource;
|
|
667
|
-
const sourceStr = sourceInfo
|
|
668
|
-
? [sourceInfo.source, sourceInfo.mode, sourceInfo.path].filter(Boolean).join('/')
|
|
669
|
-
: 'unknown';
|
|
670
|
-
const fallbackStr = sourceInfo?.fallback && sourceInfo.fallback !== 'none'
|
|
671
|
-
? ` fallback=${sourceInfo.fallback}`
|
|
672
|
-
: '';
|
|
673
|
-
const elapsedStr = sourceInfo && sourceInfo.elapsed > 0 ? ` ${sourceInfo.elapsed.toFixed(2)}s` : '';
|
|
674
|
-
if (onProgress) {
|
|
675
|
-
onProgress({
|
|
676
|
-
stage: 'shards',
|
|
677
|
-
progress: pct,
|
|
678
|
-
shard: shardsLoaded,
|
|
679
|
-
totalShards,
|
|
680
|
-
bytesLoaded,
|
|
681
|
-
totalBytes,
|
|
682
|
-
bytesPerSecond: speed,
|
|
683
|
-
message: `Shard ${shardIndex}: ${sourceStr} (${formatBytes(shardSize)}${elapsedStr}${fallbackStr})`,
|
|
684
|
-
});
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
}
|
|
688
|
-
return data;
|
|
689
|
-
};
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
let loadError = null;
|
|
693
|
-
try {
|
|
694
|
-
reportProgress('shards', 0.1, 'Loading embeddings...');
|
|
695
|
-
activeLoadPhase = 'embeddings';
|
|
696
|
-
phaseStart = nowMs();
|
|
697
|
-
await this.#loadEmbeddings(onProgress);
|
|
698
|
-
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
699
|
-
this.#assertResidentBudget('embeddings');
|
|
700
|
-
|
|
701
|
-
const resolveNumLayers = (value) => {
|
|
702
|
-
const normalized = Number(value);
|
|
703
|
-
if (!Number.isInteger(normalized) || normalized <= 0) {
|
|
704
|
-
return 0;
|
|
705
|
-
}
|
|
706
|
-
return normalized;
|
|
707
|
-
};
|
|
708
|
-
|
|
709
|
-
const manifestConfig = this.manifest.config;
|
|
710
|
-
const layerCountCandidates = [
|
|
711
|
-
manifestConfig?.num_hidden_layers,
|
|
712
|
-
manifestConfig?.blockCount,
|
|
713
|
-
manifestConfig?.text_config?.num_hidden_layers,
|
|
714
|
-
manifestConfig?.n_layer,
|
|
715
|
-
this.manifest.architecture?.numLayers,
|
|
716
|
-
];
|
|
717
|
-
const numLayers = layerCountCandidates
|
|
718
|
-
.map(resolveNumLayers)
|
|
719
|
-
.find((count) => Number.isInteger(count) && count > 0);
|
|
720
|
-
|
|
721
|
-
if (!Number.isInteger(numLayers)) {
|
|
722
|
-
throw new Error(
|
|
723
|
-
`Manifest "${this.manifest.modelId ?? 'unknown'}" missing or invalid layer count. ` +
|
|
724
|
-
`Expected one of manifest.config.num_hidden_layers/blockCount/text_config.num_hidden_layers/n_layer ` +
|
|
725
|
-
`or manifest.architecture.numLayers.`
|
|
726
|
-
);
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
log.info('Loader', `Layers: 0-${numLayers - 1}`);
|
|
730
|
-
|
|
731
|
-
inLayerPhase = true;
|
|
732
|
-
activeLoadPhase = 'layers';
|
|
733
|
-
const layersStartTime = performance.now();
|
|
734
|
-
let layerTotalMs = 0;
|
|
735
|
-
let maxLayerMs = 0;
|
|
736
|
-
let maxLayer = null;
|
|
737
|
-
|
|
738
|
-
for (let l = 0; l < numLayers; l++) {
|
|
739
|
-
const layerStart = performance.now();
|
|
740
|
-
const layerPromise = this.#loadLayer(l, onProgress);
|
|
741
|
-
this.#prefetchLayerShards(l);
|
|
742
|
-
await layerPromise;
|
|
743
|
-
const layerElapsedMs = performance.now() - layerStart;
|
|
744
|
-
layerTotalMs += layerElapsedMs;
|
|
745
|
-
if (layerElapsedMs > maxLayerMs) {
|
|
746
|
-
maxLayerMs = layerElapsedMs;
|
|
747
|
-
maxLayer = l;
|
|
748
|
-
}
|
|
749
|
-
const layerElapsed = (layerElapsedMs / 1000).toFixed(2);
|
|
750
|
-
log.verbose('Loader', ` Layer ${l}: ${layerElapsed}s`);
|
|
751
|
-
|
|
752
|
-
await new Promise(r => setTimeout(r, 0));
|
|
753
|
-
|
|
754
|
-
const { flushIntervalLayers, flushThresholdBytes, gpuQueueFlushLayers } = this.#loadingConfig.memoryManagement;
|
|
755
|
-
const cacheBytes = this.shardCache.totalBytes;
|
|
756
|
-
const shouldFlushCache = !this.shardCache.hasCustomLoader && l > 0 && (l % flushIntervalLayers === 0 || cacheBytes > flushThresholdBytes);
|
|
757
|
-
if (shouldFlushCache) {
|
|
758
|
-
this.shardCache.clear();
|
|
759
|
-
}
|
|
760
|
-
if (l > 0 && l % gpuQueueFlushLayers === 0) {
|
|
761
|
-
const device = getDevice();
|
|
762
|
-
if (device) {
|
|
763
|
-
await device.queue.onSubmittedWorkDone();
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
|
|
767
|
-
if (onProgress) {
|
|
768
|
-
syncCustomReadStats();
|
|
769
|
-
const layerFraction = (l + 1) / numLayers;
|
|
770
|
-
const layerProgress = 0.80 + layerFraction * 0.05;
|
|
771
|
-
onProgress({
|
|
772
|
-
stage: 'layers',
|
|
773
|
-
layer: l + 1,
|
|
774
|
-
total: numLayers,
|
|
775
|
-
progress: layerProgress,
|
|
776
|
-
shard: shardsLoaded,
|
|
777
|
-
totalShards,
|
|
778
|
-
bytesLoaded,
|
|
779
|
-
totalBytes,
|
|
780
|
-
bytesPerSecond: 0,
|
|
781
|
-
message: `Layer ${l + 1}/${numLayers}`,
|
|
782
|
-
});
|
|
783
|
-
}
|
|
784
|
-
this.#assertResidentBudget(`layer ${l + 1}`);
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
const layersTotalTime = ((performance.now() - layersStartTime) / 1000).toFixed(2);
|
|
788
|
-
this.loadTiming.layers = {
|
|
789
|
-
count: numLayers,
|
|
790
|
-
totalMs: roundLoadTimingMs(layerTotalMs),
|
|
791
|
-
meanMs: numLayers > 0 ? roundLoadTimingMs(layerTotalMs / numLayers) : null,
|
|
792
|
-
maxMs: maxLayer == null ? null : roundLoadTimingMs(maxLayerMs),
|
|
793
|
-
maxLayer,
|
|
794
|
-
};
|
|
795
|
-
finishLoadPhase(this.loadTiming, activeLoadPhase, layersStartTime);
|
|
796
|
-
log.info('Loader', `Layers: ${numLayers} complete (${layersTotalTime}s)`);
|
|
797
|
-
|
|
798
|
-
reportProgress('gpu_transfer', 0.85, 'Loading final weights...');
|
|
799
|
-
activeLoadPhase = 'finalWeights';
|
|
800
|
-
phaseStart = nowMs();
|
|
801
|
-
await this.#loadFinalWeights(onProgress);
|
|
802
|
-
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
803
|
-
this.#assertResidentBudget('final weights');
|
|
804
|
-
syncCustomReadStats();
|
|
805
|
-
|
|
806
|
-
if (onProgress) {
|
|
807
|
-
onProgress({
|
|
808
|
-
stage: 'complete',
|
|
809
|
-
progress: 1.0,
|
|
810
|
-
shard: shardsLoaded,
|
|
811
|
-
totalShards,
|
|
812
|
-
bytesLoaded,
|
|
813
|
-
totalBytes,
|
|
814
|
-
});
|
|
815
|
-
}
|
|
816
|
-
|
|
817
|
-
this.isLoaded = true;
|
|
818
|
-
const totalTime = ((Date.now() - loadStartTime) / 1000).toFixed(2);
|
|
819
|
-
const avgSpeed = formatBytes(bytesLoaded / (Date.now() - loadStartTime) * 1000);
|
|
820
|
-
log.info('Loader', `Complete: ${formatBytes(bytesLoaded)} in ${totalTime}s (${avgSpeed}/s)`);
|
|
821
|
-
|
|
822
|
-
activeLoadPhase = 'cleanup';
|
|
823
|
-
phaseStart = nowMs();
|
|
824
|
-
this.shardCache.clear();
|
|
825
|
-
finishLoadPhase(this.loadTiming, activeLoadPhase, phaseStart);
|
|
826
|
-
finishLoadTiming(this.loadTiming, 'complete', loadTimingStart);
|
|
827
|
-
|
|
828
|
-
return (this.manifest.config) || {};
|
|
829
|
-
} catch (error) {
|
|
830
|
-
loadError = error;
|
|
831
|
-
syncCustomReadStats();
|
|
832
|
-
finishLoadTiming(this.loadTiming, 'failed', loadTimingStart, error, activeLoadPhase);
|
|
833
|
-
} finally {
|
|
834
|
-
this.#loadShardOverride = null;
|
|
835
|
-
if (this.#memoryMonitor) {
|
|
836
|
-
this.#stopMemoryLogging(loadError ? 'failed' : 'complete');
|
|
837
|
-
}
|
|
838
|
-
}
|
|
839
|
-
|
|
840
|
-
if (loadError) {
|
|
841
|
-
await this.unload();
|
|
842
|
-
if (preservedManifest) {
|
|
843
|
-
this.manifest = preservedManifest;
|
|
844
|
-
}
|
|
845
|
-
throw loadError;
|
|
846
|
-
}
|
|
847
|
-
return (this.manifest?.config) || {};
|
|
407
|
+
return loadImpl.call(this, modelId, options);
|
|
848
408
|
}
|
|
849
409
|
|
|
850
410
|
|
|
851
|
-
async
|
|
411
|
+
async _buildTensorLocations() {
|
|
852
412
|
this.tensorLocations.clear();
|
|
853
413
|
if (!this.manifest) {
|
|
854
|
-
this
|
|
414
|
+
this._layerShardMap.clear();
|
|
855
415
|
return;
|
|
856
416
|
}
|
|
857
417
|
|
|
858
418
|
const locations = await buildTensorLocations(this.manifest, {
|
|
859
|
-
tensorsJsonUrl: this
|
|
860
|
-
loadTensorsJson: this
|
|
419
|
+
tensorsJsonUrl: this._tensorsJsonUrl,
|
|
420
|
+
loadTensorsJson: this._loadTensorsJson,
|
|
861
421
|
hasCustomLoader: this.shardCache.hasCustomLoader,
|
|
862
422
|
});
|
|
863
423
|
|
|
@@ -865,11 +425,11 @@ export class DopplerLoader {
|
|
|
865
425
|
this.tensorLocations.set(name, loc);
|
|
866
426
|
}
|
|
867
427
|
|
|
868
|
-
this
|
|
428
|
+
this._buildLayerShardMap();
|
|
869
429
|
}
|
|
870
430
|
|
|
871
|
-
|
|
872
|
-
this
|
|
431
|
+
_buildLayerShardMap() {
|
|
432
|
+
this._layerShardMap.clear();
|
|
873
433
|
|
|
874
434
|
for (const [, location] of this.tensorLocations) {
|
|
875
435
|
const layerIdx = getLayerIndexFromGroup(location.group);
|
|
@@ -877,10 +437,10 @@ export class DopplerLoader {
|
|
|
877
437
|
continue;
|
|
878
438
|
}
|
|
879
439
|
|
|
880
|
-
let shards = this
|
|
440
|
+
let shards = this._layerShardMap.get(layerIdx);
|
|
881
441
|
if (!shards) {
|
|
882
442
|
shards = new Set();
|
|
883
|
-
this
|
|
443
|
+
this._layerShardMap.set(layerIdx, shards);
|
|
884
444
|
}
|
|
885
445
|
|
|
886
446
|
for (const shardIndex of getTensorShardIndices(location)) {
|
|
@@ -889,7 +449,7 @@ export class DopplerLoader {
|
|
|
889
449
|
}
|
|
890
450
|
}
|
|
891
451
|
|
|
892
|
-
|
|
452
|
+
_logWeightBreakdown() {
|
|
893
453
|
if (this.tensorLocations.size === 0) return;
|
|
894
454
|
|
|
895
455
|
let totalBytes = 0;
|
|
@@ -911,8 +471,8 @@ export class DopplerLoader {
|
|
|
911
471
|
}
|
|
912
472
|
}
|
|
913
473
|
|
|
914
|
-
|
|
915
|
-
const prefetch = this
|
|
474
|
+
_prefetchLayerShards(layerIdx) {
|
|
475
|
+
const prefetch = this._loadingConfig.prefetch;
|
|
916
476
|
if (!prefetch?.enabled) return;
|
|
917
477
|
// Range-capable custom loaders are expected to serve fine-grained tensor reads.
|
|
918
478
|
// Whole-shard prefetch defeats that contract and can force invalid >4 GiB reads
|
|
@@ -934,15 +494,15 @@ export class DopplerLoader {
|
|
|
934
494
|
|
|
935
495
|
const layersAhead = prefetch.layersAhead;
|
|
936
496
|
if (!Number.isFinite(layersAhead) || layersAhead <= 0) return;
|
|
937
|
-
if (this
|
|
497
|
+
if (this._layerShardMap.size === 0) return;
|
|
938
498
|
|
|
939
499
|
const maxShards = prefetch.maxShards;
|
|
940
500
|
const hasLimit = maxShards > 0;
|
|
941
501
|
let scheduled = 0;
|
|
942
|
-
const loadShard = this
|
|
502
|
+
const loadShard = this._getLoadShard();
|
|
943
503
|
|
|
944
504
|
for (let idx = layerIdx + 1; idx <= layerIdx + layersAhead; idx++) {
|
|
945
|
-
const shards = this
|
|
505
|
+
const shards = this._layerShardMap.get(idx);
|
|
946
506
|
if (!shards) continue;
|
|
947
507
|
|
|
948
508
|
for (const shardIndex of shards) {
|
|
@@ -956,402 +516,70 @@ export class DopplerLoader {
|
|
|
956
516
|
}
|
|
957
517
|
|
|
958
518
|
|
|
959
|
-
async
|
|
960
|
-
|
|
961
|
-
if (!location) {
|
|
962
|
-
if (!silent) {
|
|
963
|
-
log.warn('Loader', `Tensor not found: ${name}`);
|
|
964
|
-
}
|
|
965
|
-
return null;
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
if (name.includes('attn_k') || name.includes('k_proj')) {
|
|
969
|
-
debugTrace.loader(`Loading ${name}: shape=${JSON.stringify(location.shape)}, size=${location.size}, dtype=${location.dtype}, spans=${!!location.spans}`);
|
|
970
|
-
}
|
|
971
|
-
|
|
972
|
-
const streamedUpload = toGPU && this.#shouldStreamUploadToGPU(location);
|
|
973
|
-
let shardData;
|
|
974
|
-
try {
|
|
975
|
-
const preserveRawSourceBytes = toGPU && isLiteRTAffineInt4FusedEligible(location, {
|
|
976
|
-
gpuCapabilities: this.gpuCapabilities,
|
|
977
|
-
});
|
|
978
|
-
shardData = this.#isFunctionalDescriptorLocation(location)
|
|
979
|
-
? await this.#assembleFunctionalDescriptorData(location, name)
|
|
980
|
-
: streamedUpload
|
|
981
|
-
? await this.#assembleShardDataToGpuBuffer(location, name)
|
|
982
|
-
: await this.#assembleShardData(location, name, {
|
|
983
|
-
materializeSourceTransform: !preserveRawSourceBytes,
|
|
984
|
-
});
|
|
985
|
-
} catch (error) {
|
|
986
|
-
throw annotateTensorLoadError(error, name, location, {
|
|
987
|
-
tensorLoadStage: streamedUpload ? 'streamShardToGpuBuffer' : 'assembleShardData',
|
|
988
|
-
toGPU,
|
|
989
|
-
streamedUpload,
|
|
990
|
-
});
|
|
991
|
-
}
|
|
992
|
-
|
|
993
|
-
if (toGPU) {
|
|
994
|
-
const device = getDevice();
|
|
995
|
-
if (!device) {
|
|
996
|
-
log.warn('Loader', 'GPU device not available; falling back to CPU');
|
|
997
|
-
if (isGpuBufferInstance(shardData)) {
|
|
998
|
-
releaseBuffer(shardData);
|
|
999
|
-
shardData = await this.#assembleShardData(location, name);
|
|
1000
|
-
}
|
|
1001
|
-
return loadTensorToCPU(shardData, location, name);
|
|
1002
|
-
}
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
const allowF32UpcastNonMatmul = this.#loadingConfig?.allowF32UpcastNonMatmul;
|
|
1006
|
-
if (allowF32UpcastNonMatmul == null) {
|
|
1007
|
-
throw new Error('runtime.loading.allowF32UpcastNonMatmul is required.');
|
|
1008
|
-
}
|
|
1009
|
-
const config = {
|
|
1010
|
-
useFusedQ4K: this.useFusedQ4K,
|
|
1011
|
-
q4kMaterializationMode: this.q4kMaterializationMode,
|
|
1012
|
-
q4kFusedRoles: this.q4kFusedRoles,
|
|
1013
|
-
keepF32Weights: this.keepF32Weights,
|
|
1014
|
-
q4kLayout: this.q4kLayout,
|
|
1015
|
-
loaderDebug: this.#loaderDebug,
|
|
1016
|
-
gpuCapabilities: this.gpuCapabilities,
|
|
1017
|
-
allowF32UpcastNonMatmul,
|
|
1018
|
-
};
|
|
1019
|
-
|
|
1020
|
-
let result;
|
|
1021
|
-
try {
|
|
1022
|
-
result = await loadTensorToGPU(shardData, location, name, config);
|
|
1023
|
-
} catch (error) {
|
|
1024
|
-
if (isGpuBufferInstance(shardData)) {
|
|
1025
|
-
releaseBuffer(shardData);
|
|
1026
|
-
}
|
|
1027
|
-
throw annotateTensorLoadError(error, name, location, {
|
|
1028
|
-
tensorLoadStage: 'materializeTensorToGPU',
|
|
1029
|
-
toGPU: true,
|
|
1030
|
-
streamedUpload,
|
|
1031
|
-
});
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
for (const buffer of result.allocatedBuffers) {
|
|
1035
|
-
this.gpuBuffers.add(buffer);
|
|
1036
|
-
}
|
|
1037
|
-
|
|
1038
|
-
return result.data;
|
|
1039
|
-
}
|
|
1040
|
-
|
|
1041
|
-
if (isGpuBufferInstance(shardData)) {
|
|
1042
|
-
// Shouldn't happen (streaming is only used for toGPU), but keep this leak-proof.
|
|
1043
|
-
releaseBuffer(shardData);
|
|
1044
|
-
shardData = await this.#assembleShardData(location, name);
|
|
1045
|
-
}
|
|
1046
|
-
return loadTensorToCPU(shardData, location, name);
|
|
519
|
+
async _loadTensor(name, toGPU = true, silent = false) {
|
|
520
|
+
return _loadTensorImpl.call(this, name, toGPU, silent);
|
|
1047
521
|
}
|
|
1048
522
|
|
|
1049
|
-
|
|
523
|
+
_isFunctionalDescriptorLocation(location) {
|
|
1050
524
|
return String(location?.dtype || '').trim().toUpperCase() === 'FUNCTIONAL_DESCRIPTOR';
|
|
1051
525
|
}
|
|
1052
526
|
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
throw new Error(
|
|
1056
|
-
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" is missing descriptorManifest.`
|
|
1057
|
-
);
|
|
1058
|
-
}
|
|
1059
|
-
const manifest = assertFunctionalDescriptorManifest(
|
|
1060
|
-
location.descriptorManifest,
|
|
1061
|
-
`FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptorManifest`
|
|
1062
|
-
);
|
|
1063
|
-
const components = manifest.components;
|
|
1064
|
-
if (!components || typeof components !== 'object') {
|
|
1065
|
-
throw new Error(
|
|
1066
|
-
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptorManifest.components is required.`
|
|
1067
|
-
);
|
|
1068
|
-
}
|
|
1069
|
-
const files = [
|
|
1070
|
-
components.kronecker_sum?.shard_file,
|
|
1071
|
-
components.coordinate_inr?.shard_file,
|
|
1072
|
-
components.sparse_outliers?.shard_file,
|
|
1073
|
-
];
|
|
1074
|
-
if (files.some((file) => typeof file !== 'string' || file.trim().length === 0)) {
|
|
1075
|
-
throw new Error(
|
|
1076
|
-
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" must declare kronecker, SIREN, and sparse shard_file values.`
|
|
1077
|
-
);
|
|
1078
|
-
}
|
|
1079
|
-
return files.map((file) => file.trim());
|
|
527
|
+
_getDescriptorShardFiles(location, name) {
|
|
528
|
+
return _getDescriptorShardFilesImpl.call(this, location, name);
|
|
1080
529
|
}
|
|
1081
530
|
|
|
1082
|
-
async
|
|
1083
|
-
|
|
1084
|
-
const payload = loadAuxiliaryFile
|
|
1085
|
-
? await loadAuxiliaryFile(file)
|
|
1086
|
-
: await loadAuxFile(file);
|
|
1087
|
-
if (payload == null) {
|
|
1088
|
-
throw new Error(
|
|
1089
|
-
`[DopplerLoader] Descriptor shard "${file}" for tensor "${name}" was not found.`
|
|
1090
|
-
);
|
|
1091
|
-
}
|
|
1092
|
-
if (payload instanceof Uint8Array) {
|
|
1093
|
-
return payload;
|
|
1094
|
-
}
|
|
1095
|
-
if (ArrayBuffer.isView(payload)) {
|
|
1096
|
-
return new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength);
|
|
1097
|
-
}
|
|
1098
|
-
if (payload instanceof ArrayBuffer) {
|
|
1099
|
-
return new Uint8Array(payload);
|
|
1100
|
-
}
|
|
1101
|
-
throw new Error(
|
|
1102
|
-
`[DopplerLoader] Descriptor shard "${file}" for tensor "${name}" must load as ArrayBuffer or Uint8Array.`
|
|
1103
|
-
);
|
|
531
|
+
async _loadDescriptorShardFile(file, name) {
|
|
532
|
+
return _loadDescriptorShardFileImpl.call(this, file, name);
|
|
1104
533
|
}
|
|
1105
534
|
|
|
1106
|
-
async
|
|
1107
|
-
|
|
1108
|
-
if (typeof descriptorHash !== 'string' || !descriptorHash.trim()) {
|
|
1109
|
-
return;
|
|
1110
|
-
}
|
|
1111
|
-
const match = /^sha256:([a-f0-9]{64})$/i.exec(descriptorHash.trim());
|
|
1112
|
-
if (!match) {
|
|
1113
|
-
throw new Error(
|
|
1114
|
-
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptor_hash must be sha256:<64 hex chars>.`
|
|
1115
|
-
);
|
|
1116
|
-
}
|
|
1117
|
-
const totalBytes = Array.from(descriptorShards.values()).reduce((sum, bytes) => sum + bytes.byteLength, 0);
|
|
1118
|
-
const combined = new Uint8Array(totalBytes);
|
|
1119
|
-
let offset = 0;
|
|
1120
|
-
for (const bytes of descriptorShards.values()) {
|
|
1121
|
-
combined.set(bytes, offset);
|
|
1122
|
-
offset += bytes.byteLength;
|
|
1123
|
-
}
|
|
1124
|
-
const actual = await computeHash(combined, 'sha256');
|
|
1125
|
-
if (actual.toLowerCase() !== match[1].toLowerCase()) {
|
|
1126
|
-
throw new Error(
|
|
1127
|
-
`[DopplerLoader] FUNCTIONAL_DESCRIPTOR tensor "${name}" descriptor hash mismatch. ` +
|
|
1128
|
-
`Expected ${descriptorHash}, got sha256:${actual}.`
|
|
1129
|
-
);
|
|
1130
|
-
}
|
|
535
|
+
async _assertDescriptorHash(location, name, descriptorShards) {
|
|
536
|
+
return _assertDescriptorHashImpl.call(this, location, name, descriptorShards);
|
|
1131
537
|
}
|
|
1132
538
|
|
|
1133
|
-
async
|
|
1134
|
-
|
|
1135
|
-
const descriptorShards = new Map();
|
|
1136
|
-
for (const file of shardFiles) {
|
|
1137
|
-
descriptorShards.set(file, await this.#loadDescriptorShardFile(file, name));
|
|
1138
|
-
}
|
|
1139
|
-
await this.#assertDescriptorHash(location, name, descriptorShards);
|
|
1140
|
-
const data = new Uint8Array(0);
|
|
1141
|
-
Object.defineProperty(data, 'descriptorShards', {
|
|
1142
|
-
value: descriptorShards,
|
|
1143
|
-
enumerable: false,
|
|
1144
|
-
});
|
|
1145
|
-
return data;
|
|
539
|
+
async _assembleFunctionalDescriptorData(location, name) {
|
|
540
|
+
return _assembleFunctionalDescriptorDataImpl.call(this, location, name);
|
|
1146
541
|
}
|
|
1147
542
|
|
|
1148
543
|
|
|
1149
|
-
async
|
|
1150
|
-
|
|
1151
|
-
const loadShardRange = (idx, offset, length) => this.shardCache.loadRange(idx, offset, length);
|
|
1152
|
-
const data = await assembleShardData(location, name, loadShard, loadShardRange, options);
|
|
1153
|
-
const companions = Array.isArray(location?.storage?.companions)
|
|
1154
|
-
? location.storage.companions
|
|
1155
|
-
: [];
|
|
1156
|
-
if (companions.length === 0) {
|
|
1157
|
-
return data;
|
|
1158
|
-
}
|
|
1159
|
-
const storageCompanions = {};
|
|
1160
|
-
for (const companion of companions) {
|
|
1161
|
-
const companionLocation = this.tensorLocations.get(companion.tensorId);
|
|
1162
|
-
if (!companionLocation) {
|
|
1163
|
-
throw new Error(
|
|
1164
|
-
`[DopplerLoader] Tensor "${name}" storage companion "${companion.tensorId}" for role "${companion.role}" was not found.`
|
|
1165
|
-
);
|
|
1166
|
-
}
|
|
1167
|
-
storageCompanions[companion.role] = {
|
|
1168
|
-
tensorId: companion.tensorId,
|
|
1169
|
-
location: companionLocation,
|
|
1170
|
-
bytes: await assembleShardData(companionLocation, companion.tensorId, loadShard, loadShardRange),
|
|
1171
|
-
};
|
|
1172
|
-
}
|
|
1173
|
-
Object.defineProperty(data, 'storageCompanions', {
|
|
1174
|
-
value: storageCompanions,
|
|
1175
|
-
enumerable: false,
|
|
1176
|
-
});
|
|
1177
|
-
return data;
|
|
544
|
+
async _assembleShardData(location, name, options = {}) {
|
|
545
|
+
return _assembleShardDataImpl.call(this, location, name, options);
|
|
1178
546
|
}
|
|
1179
547
|
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
if (!location?.size || location.size <= 0) return false;
|
|
1183
|
-
if (hasSourceTransform(location)) return false;
|
|
1184
|
-
if (Array.isArray(location?.storage?.companions) && location.storage.companions.length > 0) return false;
|
|
1185
|
-
if (requiresCpuF16ToF32MatmulMaterialization(location, this.gpuCapabilities, this.keepF32Weights)) return false;
|
|
1186
|
-
if (this.shardCache.hasCustomLoader && !this.shardCache.canStreamRanges) return false;
|
|
1187
|
-
const chunkBytes = this.#loadingConfig?.storage?.backend?.streaming?.readChunkBytes ?? 0;
|
|
1188
|
-
if (!Number.isFinite(chunkBytes) || chunkBytes <= 0) return false;
|
|
1189
|
-
// Always stream multi-span tensors to avoid loading whole shards + assembling on CPU.
|
|
1190
|
-
if (location.spans && location.spans.length > 0) {
|
|
1191
|
-
return true;
|
|
1192
|
-
}
|
|
1193
|
-
// Conservative default: only stream "large" single-span tensors to avoid turning
|
|
1194
|
-
// OPFS into many small random reads that can be slower than whole-shard caching.
|
|
1195
|
-
const minStreamBytes = Math.max(16 * 1024 * 1024, chunkBytes * 4);
|
|
1196
|
-
return location.size >= minStreamBytes;
|
|
548
|
+
_shouldStreamUploadToGPU(location) {
|
|
549
|
+
return _shouldStreamUploadToGPUImpl.call(this, location);
|
|
1197
550
|
}
|
|
1198
551
|
|
|
1199
|
-
async
|
|
1200
|
-
|
|
1201
|
-
if (!device) {
|
|
1202
|
-
throw new Error('GPU device not available');
|
|
1203
|
-
}
|
|
1204
|
-
const rawChunkBytes = Number(this.#loadingConfig?.storage?.backend?.streaming?.readChunkBytes);
|
|
1205
|
-
const chunkBytes = Number.isFinite(rawChunkBytes) && rawChunkBytes > 0
|
|
1206
|
-
? Math.floor(rawChunkBytes)
|
|
1207
|
-
: 1;
|
|
1208
|
-
|
|
1209
|
-
// queue.writeBuffer requires 4-byte aligned sizes; we pad the buffer.
|
|
1210
|
-
const alignedSize = Math.ceil(location.size / 4) * 4;
|
|
1211
|
-
const raw = acquireBuffer(alignedSize, undefined, `raw_${name}`);
|
|
1212
|
-
let complete = false;
|
|
1213
|
-
|
|
1214
|
-
try {
|
|
1215
|
-
let dstOffset = 0;
|
|
1216
|
-
let pendingBytes = null;
|
|
1217
|
-
const writeAlignedChunk = (bytes) => {
|
|
1218
|
-
if (bytes.byteLength === 0) return;
|
|
1219
|
-
device.queue.writeBuffer(raw, dstOffset, bytes, 0, bytes.byteLength);
|
|
1220
|
-
dstOffset += bytes.byteLength;
|
|
1221
|
-
};
|
|
1222
|
-
const uploadChunk = (bytes) => {
|
|
1223
|
-
let merged = bytes;
|
|
1224
|
-
if (pendingBytes && pendingBytes.byteLength > 0) {
|
|
1225
|
-
merged = new Uint8Array(pendingBytes.byteLength + bytes.byteLength);
|
|
1226
|
-
merged.set(pendingBytes, 0);
|
|
1227
|
-
merged.set(bytes, pendingBytes.byteLength);
|
|
1228
|
-
pendingBytes = null;
|
|
1229
|
-
}
|
|
1230
|
-
const alignedLength = merged.byteLength - (merged.byteLength % 4);
|
|
1231
|
-
if (alignedLength > 0) {
|
|
1232
|
-
writeAlignedChunk(merged.subarray(0, alignedLength));
|
|
1233
|
-
}
|
|
1234
|
-
const remainder = merged.byteLength - alignedLength;
|
|
1235
|
-
pendingBytes = remainder > 0 ? merged.slice(alignedLength) : null;
|
|
1236
|
-
};
|
|
1237
|
-
const streamRange = (idx, offset, length) => this.shardCache.streamRange(idx, offset, length, { chunkBytes });
|
|
1238
|
-
|
|
1239
|
-
if (location.spans) {
|
|
1240
|
-
for (const span of location.spans) {
|
|
1241
|
-
for await (const chunk of streamRange(span.shardIndex, span.offset, span.size)) {
|
|
1242
|
-
uploadChunk(chunk);
|
|
1243
|
-
}
|
|
1244
|
-
}
|
|
1245
|
-
} else {
|
|
1246
|
-
for await (const chunk of streamRange(location.shardIndex, location.offset, location.size)) {
|
|
1247
|
-
uploadChunk(chunk);
|
|
1248
|
-
}
|
|
1249
|
-
}
|
|
1250
|
-
|
|
1251
|
-
if (pendingBytes && pendingBytes.byteLength > 0) {
|
|
1252
|
-
const padded = new Uint8Array(4);
|
|
1253
|
-
padded.set(pendingBytes, 0);
|
|
1254
|
-
writeAlignedChunk(padded);
|
|
1255
|
-
dstOffset -= (4 - pendingBytes.byteLength);
|
|
1256
|
-
pendingBytes = null;
|
|
1257
|
-
}
|
|
1258
|
-
|
|
1259
|
-
if (dstOffset !== location.size) {
|
|
1260
|
-
throw new Error(
|
|
1261
|
-
`Stream upload short read for "${name}": got=${dstOffset}, expected=${location.size}.`
|
|
1262
|
-
);
|
|
1263
|
-
}
|
|
1264
|
-
complete = true;
|
|
1265
|
-
return raw;
|
|
1266
|
-
} finally {
|
|
1267
|
-
if (!complete) {
|
|
1268
|
-
releaseBuffer(raw);
|
|
1269
|
-
}
|
|
1270
|
-
}
|
|
552
|
+
async _assembleShardDataToGpuBuffer(location, name) {
|
|
553
|
+
return _assembleShardDataToGpuBufferImpl.call(this, location, name);
|
|
1271
554
|
}
|
|
1272
555
|
|
|
1273
556
|
|
|
1274
|
-
|
|
557
|
+
_needsNormWeightOffset() {
|
|
1275
558
|
const result = needsNormWeightOffset(this.manifest);
|
|
1276
|
-
if (result && !this
|
|
1277
|
-
this
|
|
559
|
+
if (result && !this._normOffsetLogged) {
|
|
560
|
+
this._normOffsetLogged = true;
|
|
1278
561
|
}
|
|
1279
562
|
return result;
|
|
1280
563
|
}
|
|
1281
564
|
|
|
1282
565
|
|
|
1283
|
-
async
|
|
1284
|
-
|
|
1285
|
-
const ctx = {
|
|
1286
|
-
tensorLocations: this.tensorLocations,
|
|
1287
|
-
loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
|
|
1288
|
-
loadShardRange: (index, offset, length) => this.shardCache.loadRange(index, offset, length),
|
|
1289
|
-
shouldStreamLargeWeight: (name, loc, label) => this.#shouldStreamLargeWeight(name, loc, label),
|
|
1290
|
-
resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
|
|
1291
|
-
gpuBuffers: this.gpuBuffers,
|
|
1292
|
-
keepF32Weights: this.keepF32Weights,
|
|
1293
|
-
// Keep embedding weights in F32 when manifest quantization requires it.
|
|
1294
|
-
// gather.wgsl reads embeddings as f32; downcasting here corrupts reads.
|
|
1295
|
-
preserveF32Embeddings: String(this.manifest?.quantizationInfo?.embeddings ?? '').toLowerCase() === 'f32',
|
|
1296
|
-
hostHasShaderF16: this.gpuCapabilities?.hasF16 ?? null,
|
|
1297
|
-
embeddingKernel: this.manifest?.inference?.execution?.kernels?.embed ?? null,
|
|
1298
|
-
};
|
|
1299
|
-
|
|
1300
|
-
this.embeddings = await loadEmbeddings(ctx);
|
|
1301
|
-
this.perLayerInputWeights = await loadPerLayerInputWeights({
|
|
1302
|
-
modelId: this.manifest?.modelId ?? null,
|
|
1303
|
-
tensorLocations: this.tensorLocations,
|
|
1304
|
-
gpuBuffers: this.gpuBuffers,
|
|
1305
|
-
loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
|
|
1306
|
-
shouldStreamLargeWeight: (name, loc, label) => this.#shouldStreamLargeWeight(name, loc, label),
|
|
1307
|
-
loadShardRange: (index, offset, length) => this.shardCache.loadRange(index, offset, length),
|
|
1308
|
-
resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
|
|
1309
|
-
perLayerInputSession: this.#perLayerInputSession,
|
|
1310
|
-
}, this.manifest?.architecture ?? null);
|
|
566
|
+
async _loadEmbeddings(_onProgress) {
|
|
567
|
+
return _loadEmbeddingsImpl.call(this, _onProgress);
|
|
1311
568
|
}
|
|
1312
569
|
|
|
1313
570
|
|
|
1314
|
-
async
|
|
1315
|
-
|
|
1316
|
-
this.manifest?.config?.text_config
|
|
1317
|
-
&& typeof this.manifest.config.text_config === 'object'
|
|
1318
|
-
&& !Array.isArray(this.manifest.config.text_config)
|
|
1319
|
-
)
|
|
1320
|
-
? this.manifest.config.text_config
|
|
1321
|
-
: this.manifest?.config ?? null;
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
const ctx = {
|
|
1325
|
-
tensorLocations: this.tensorLocations,
|
|
1326
|
-
loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
|
|
1327
|
-
needsNormWeightOffset: () => this.#needsNormWeightOffset(),
|
|
1328
|
-
gpuBuffers: this.gpuBuffers,
|
|
1329
|
-
keepF32Weights: this.keepF32Weights,
|
|
1330
|
-
isMoE: this.isMoE,
|
|
1331
|
-
isExpertLayer: (idx) => this.#isExpertLayer(idx),
|
|
1332
|
-
loadDenseFfnForMoeLayers: this.manifest?.inference?.ffn?.branchMode === 'dense_plus_moe',
|
|
1333
|
-
numHeads: this.manifest?.architecture?.numAttentionHeads ?? null,
|
|
1334
|
-
numKVHeads: this.manifest?.architecture?.numKeyValueHeads ?? null,
|
|
1335
|
-
headDim: this.manifest?.architecture?.headDim ?? null,
|
|
1336
|
-
hiddenSize: this.manifest?.architecture?.hiddenSize ?? null,
|
|
1337
|
-
linearNumKeyHeads: textConfig?.linear_num_key_heads ?? this.manifest?.architecture?.linearNumKeyHeads ?? null,
|
|
1338
|
-
linearNumValueHeads: textConfig?.linear_num_value_heads ?? this.manifest?.architecture?.linearNumValueHeads ?? null,
|
|
1339
|
-
linearKeyHeadDim: textConfig?.linear_key_head_dim ?? this.manifest?.architecture?.linearKeyHeadDim ?? null,
|
|
1340
|
-
linearValueHeadDim: textConfig?.linear_value_head_dim ?? this.manifest?.architecture?.linearValueHeadDim ?? null,
|
|
1341
|
-
};
|
|
1342
|
-
|
|
1343
|
-
const weights = await loadLayer(ctx, layerIdx);
|
|
1344
|
-
this.layers.set(layerIdx, weights);
|
|
571
|
+
async _loadLayer(layerIdx, _onProgress) {
|
|
572
|
+
return _loadLayerImpl.call(this, layerIdx, _onProgress);
|
|
1345
573
|
}
|
|
1346
574
|
|
|
1347
575
|
|
|
1348
|
-
|
|
576
|
+
_isExpertLayer(_layerIdx) {
|
|
1349
577
|
return this.isMoE;
|
|
1350
578
|
}
|
|
1351
579
|
|
|
1352
580
|
|
|
1353
581
|
prefetchExperts(nextLayerIdx, expertIndices) {
|
|
1354
|
-
prefetchExpertsFromModule(this
|
|
582
|
+
prefetchExpertsFromModule(this._getExpertLoaderContext(), nextLayerIdx, expertIndices, this.isMoE);
|
|
1355
583
|
}
|
|
1356
584
|
|
|
1357
585
|
|
|
@@ -1361,16 +589,16 @@ export class DopplerLoader {
|
|
|
1361
589
|
|
|
1362
590
|
|
|
1363
591
|
async loadExpert(layerIdx, expertIdx) {
|
|
1364
|
-
return loadExpertFromModule(this
|
|
592
|
+
return loadExpertFromModule(this._getExpertLoaderContext(), layerIdx, expertIdx);
|
|
1365
593
|
}
|
|
1366
594
|
|
|
1367
595
|
|
|
1368
|
-
|
|
1369
|
-
const loadShard = this
|
|
596
|
+
_getExpertLoaderContext() {
|
|
597
|
+
const loadShard = this._getLoadShard();
|
|
1370
598
|
return {
|
|
1371
599
|
manifest: this.manifest,
|
|
1372
600
|
tensorLocations: this.tensorLocations,
|
|
1373
|
-
loadTensor: (name, toGPU, silent) => this
|
|
601
|
+
loadTensor: (name, toGPU, silent) => this._loadTensor(name, toGPU, silent),
|
|
1374
602
|
loadShard,
|
|
1375
603
|
shardCache: this.shardCache,
|
|
1376
604
|
expertCache: this.expertCache,
|
|
@@ -1381,44 +609,8 @@ export class DopplerLoader {
|
|
|
1381
609
|
}
|
|
1382
610
|
|
|
1383
611
|
|
|
1384
|
-
async
|
|
1385
|
-
|
|
1386
|
-
if (tieWordEmbeddings == null) {
|
|
1387
|
-
const modelId = this.manifest?.modelId ?? 'unknown';
|
|
1388
|
-
throw new Error(
|
|
1389
|
-
`Manifest "${modelId}" is missing inference.output.tieWordEmbeddings. ` +
|
|
1390
|
-
'Re-convert the model with a complete manifest.inference config.'
|
|
1391
|
-
);
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
const ctx = {
|
|
1396
|
-
tensorLocations: this.tensorLocations,
|
|
1397
|
-
loadTensor: (name, toGPU, silent) => this.#loadTensor(name, toGPU, silent),
|
|
1398
|
-
loadShardRange: (index, offset, length) => this.shardCache.loadRange(index, offset, length),
|
|
1399
|
-
needsNormWeightOffset: () => this.#needsNormWeightOffset(),
|
|
1400
|
-
shouldStreamLargeWeight: (name, loc, label) => this.#shouldStreamLargeWeight(name, loc, label),
|
|
1401
|
-
resolveWeightLayout: (loc) => this.#resolveWeightLayout(loc),
|
|
1402
|
-
embeddings: this.embeddings,
|
|
1403
|
-
embeddingPostprocessor: this.manifest?.inference?.output?.embeddingPostprocessor ?? null,
|
|
1404
|
-
finalNormBiasTensor: this.manifest?.inference?.normalization?.finalNormBiasTensor ?? null,
|
|
1405
|
-
lmHeadBiasTensor: this.manifest?.inference?.output?.lmHeadBiasTensor ?? null,
|
|
1406
|
-
diffusionGemmaSelfConditioning: this.manifest?.inference?.diffusionGemma?.selfConditioning === true,
|
|
1407
|
-
modelType: this.manifest?.modelType ?? null,
|
|
1408
|
-
tieWordEmbeddings,
|
|
1409
|
-
gpuBuffers: this.gpuBuffers,
|
|
1410
|
-
keepF32Weights: this.keepF32Weights,
|
|
1411
|
-
normOffsetDebugLogged: this.#normOffsetDebugLogged,
|
|
1412
|
-
};
|
|
1413
|
-
|
|
1414
|
-
const result = await loadFinalWeights(ctx);
|
|
1415
|
-
this.finalNorm = result.finalNorm;
|
|
1416
|
-
this.finalNormBias = result.finalNormBias;
|
|
1417
|
-
this.lmHead = result.lmHead;
|
|
1418
|
-
this.lmHeadBias = result.lmHeadBias;
|
|
1419
|
-
this.embeddingPostprocessor = result.embeddingPostprocessor;
|
|
1420
|
-
this.diffusionGemmaSelfConditioning = result.diffusionGemmaSelfConditioning;
|
|
1421
|
-
this.#normOffsetDebugLogged = result.normOffsetDebugLogged;
|
|
612
|
+
async _loadFinalWeights(_onProgress) {
|
|
613
|
+
return _loadFinalWeightsImpl.call(this, _onProgress);
|
|
1422
614
|
}
|
|
1423
615
|
|
|
1424
616
|
|
|
@@ -1426,13 +618,13 @@ export class DopplerLoader {
|
|
|
1426
618
|
return this.layers.get(layerIdx) || null;
|
|
1427
619
|
}
|
|
1428
620
|
|
|
1429
|
-
/**
|
|
1430
|
-
* Load a tensor by name. Public interface for extension loaders (e.g., vision).
|
|
1431
|
-
*/
|
|
1432
621
|
async loadTensor(name, toGPU = true, silent = false) {
|
|
1433
|
-
return this
|
|
622
|
+
return this._loadTensor(name, toGPU, silent);
|
|
1434
623
|
}
|
|
1435
624
|
|
|
625
|
+
async loadGpuTensor(name, silent = false) {
|
|
626
|
+
return loadGpuTensorImpl(this, name, silent);
|
|
627
|
+
}
|
|
1436
628
|
|
|
1437
629
|
getConfig() {
|
|
1438
630
|
return (this.manifest?.config) || {};
|
|
@@ -1471,8 +663,8 @@ export class DopplerLoader {
|
|
|
1471
663
|
async unload() {
|
|
1472
664
|
debugTrace.loader(' Unloading model...');
|
|
1473
665
|
|
|
1474
|
-
if (this
|
|
1475
|
-
this
|
|
666
|
+
if (this._memoryMonitor) {
|
|
667
|
+
this._stopMemoryLogging('complete');
|
|
1476
668
|
}
|
|
1477
669
|
|
|
1478
670
|
const releaseCandidate = (value) => {
|
|
@@ -1544,9 +736,9 @@ export class DopplerLoader {
|
|
|
1544
736
|
this.isLoaded = false;
|
|
1545
737
|
this.loadTiming = null;
|
|
1546
738
|
this.tensorLocations.clear();
|
|
1547
|
-
this
|
|
739
|
+
this._layerShardMap.clear();
|
|
1548
740
|
this.shardCache.clear();
|
|
1549
|
-
this
|
|
741
|
+
this._normOffsetLogged = false;
|
|
1550
742
|
|
|
1551
743
|
debugTrace.loader(' Model unloaded');
|
|
1552
744
|
}
|
|
@@ -1565,10 +757,8 @@ function isExpertGroup(group) {
|
|
|
1565
757
|
return group.includes('.expert.') || group.includes('.shared_expert');
|
|
1566
758
|
}
|
|
1567
759
|
|
|
1568
|
-
|
|
1569
760
|
let globalLoader = null;
|
|
1570
761
|
|
|
1571
|
-
|
|
1572
762
|
export function getDopplerLoader(loadingConfig) {
|
|
1573
763
|
if (!globalLoader) {
|
|
1574
764
|
globalLoader = new DopplerLoader(loadingConfig);
|
|
@@ -1578,7 +768,6 @@ export function getDopplerLoader(loadingConfig) {
|
|
|
1578
768
|
return globalLoader;
|
|
1579
769
|
}
|
|
1580
770
|
|
|
1581
|
-
|
|
1582
771
|
export function createDopplerLoader(loadingConfig) {
|
|
1583
772
|
return new DopplerLoader(loadingConfig);
|
|
1584
773
|
}
|