doppler-gpu 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +54 -33
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
package/src/cli/doppler-serve.js
CHANGED
|
@@ -4,13 +4,22 @@ import http from 'node:http';
|
|
|
4
4
|
import crypto from 'node:crypto';
|
|
5
5
|
import path from 'node:path';
|
|
6
6
|
import process from 'node:process';
|
|
7
|
-
import { fileURLToPath } from 'node:url';
|
|
7
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
8
8
|
import { DOPPLER_VERSION, doppler } from '../index.js';
|
|
9
|
-
import { listQuickstartModels } from '../client/doppler-registry.js';
|
|
9
|
+
import { listQuickstartModels, resolveQuickstartModel } from '../client/doppler-registry.js';
|
|
10
10
|
|
|
11
11
|
const DEFAULT_PORT = 8080;
|
|
12
12
|
const DEFAULT_HOST = '127.0.0.1';
|
|
13
13
|
|
|
14
|
+
export class ServeRequestError extends Error {
|
|
15
|
+
constructor(message, statusCode = 400, type = 'invalid_request_error') {
|
|
16
|
+
super(message);
|
|
17
|
+
this.name = 'ServeRequestError';
|
|
18
|
+
this.statusCode = statusCode;
|
|
19
|
+
this.type = type;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
14
23
|
function usage() {
|
|
15
24
|
return [
|
|
16
25
|
'Usage:',
|
|
@@ -18,6 +27,7 @@ function usage() {
|
|
|
18
27
|
'',
|
|
19
28
|
'Options:',
|
|
20
29
|
' --model <id> Pre-load a model at startup (optional, lazy-loads on request otherwise)',
|
|
30
|
+
' --model-url <url> Explicit local or remote RDRR artifact URL for --model',
|
|
21
31
|
' --port <n> Port to listen on (default: 8080)',
|
|
22
32
|
' --host <addr> Host to bind to (default: 127.0.0.1)',
|
|
23
33
|
' --help Show this help',
|
|
@@ -29,7 +39,7 @@ function usage() {
|
|
|
29
39
|
'',
|
|
30
40
|
'Examples:',
|
|
31
41
|
' node src/cli/doppler-serve.js --model gemma3-270m',
|
|
32
|
-
' node src/cli/doppler-serve.js --model
|
|
42
|
+
' node src/cli/doppler-serve.js --model gemma4-e2b --port 3000',
|
|
33
43
|
'',
|
|
34
44
|
'Then use with any OpenAI-compatible client:',
|
|
35
45
|
' curl http://localhost:8080/v1/chat/completions \\',
|
|
@@ -43,6 +53,7 @@ export function parseServeArgs(argv) {
|
|
|
43
53
|
port: DEFAULT_PORT,
|
|
44
54
|
host: DEFAULT_HOST,
|
|
45
55
|
model: null,
|
|
56
|
+
modelUrl: null,
|
|
46
57
|
help: false,
|
|
47
58
|
};
|
|
48
59
|
for (let i = 0; i < argv.length; i += 1) {
|
|
@@ -54,7 +65,7 @@ export function parseServeArgs(argv) {
|
|
|
54
65
|
if (!token.startsWith('--')) {
|
|
55
66
|
throw new Error(`Unexpected positional argument: ${token}`);
|
|
56
67
|
}
|
|
57
|
-
if (token !== '--port' && token !== '--host' && token !== '--model') {
|
|
68
|
+
if (token !== '--port' && token !== '--host' && token !== '--model' && token !== '--model-url') {
|
|
58
69
|
throw new Error(`Unknown flag ${token}.`);
|
|
59
70
|
}
|
|
60
71
|
const nextValue = argv[i + 1];
|
|
@@ -80,24 +91,81 @@ export function parseServeArgs(argv) {
|
|
|
80
91
|
i += 1;
|
|
81
92
|
continue;
|
|
82
93
|
}
|
|
94
|
+
if (token === '--model-url') {
|
|
95
|
+
flags.modelUrl = normalizeModelUrl(nextValue.trim());
|
|
96
|
+
i += 1;
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
83
99
|
throw new Error(`Unknown flag ${token}.`);
|
|
84
100
|
}
|
|
101
|
+
if (flags.modelUrl && !flags.model) {
|
|
102
|
+
throw new Error('--model-url requires --model so the served model identity remains explicit.');
|
|
103
|
+
}
|
|
85
104
|
return flags;
|
|
86
105
|
}
|
|
87
106
|
|
|
107
|
+
function normalizeModelUrl(value) {
|
|
108
|
+
if (!value) {
|
|
109
|
+
throw new Error('--model-url must be non-empty.');
|
|
110
|
+
}
|
|
111
|
+
if (/^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value)) {
|
|
112
|
+
return value;
|
|
113
|
+
}
|
|
114
|
+
return pathToFileURL(path.resolve(value)).href;
|
|
115
|
+
}
|
|
116
|
+
|
|
88
117
|
function generateCompletionId() {
|
|
89
118
|
return `chatcmpl-${crypto.randomBytes(12).toString('base64url')}`;
|
|
90
119
|
}
|
|
91
120
|
|
|
92
|
-
function
|
|
93
|
-
|
|
121
|
+
function sha256Hex(value) {
|
|
122
|
+
return crypto.createHash('sha256').update(value).digest('hex');
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function utf8ByteLength(value) {
|
|
126
|
+
return Buffer.byteLength(String(value ?? ''), 'utf8');
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function stableJson(value) {
|
|
130
|
+
if (value === null || typeof value !== 'object') {
|
|
131
|
+
return JSON.stringify(value);
|
|
132
|
+
}
|
|
133
|
+
if (Array.isArray(value)) {
|
|
134
|
+
return `[${value.map((entry) => stableJson(entry)).join(',')}]`;
|
|
135
|
+
}
|
|
136
|
+
const keys = Object.keys(value).sort();
|
|
137
|
+
const entries = keys
|
|
138
|
+
.filter((key) => value[key] !== undefined)
|
|
139
|
+
.map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`);
|
|
140
|
+
return `{${entries.join(',')}}`;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function buildSha256Evidence(value) {
|
|
144
|
+
const text = String(value ?? '');
|
|
145
|
+
return {
|
|
146
|
+
algorithm: 'sha256',
|
|
147
|
+
value: sha256Hex(text),
|
|
148
|
+
bytes: utf8ByteLength(text),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function buildJsonSha256Evidence(value) {
|
|
153
|
+
return buildSha256Evidence(stableJson(value));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function jsonError(res, statusCode, message, type, extra = null) {
|
|
157
|
+
const payload = {
|
|
94
158
|
error: {
|
|
95
159
|
message,
|
|
96
160
|
type: type ?? 'invalid_request_error',
|
|
97
161
|
param: null,
|
|
98
162
|
code: null,
|
|
99
163
|
},
|
|
100
|
-
}
|
|
164
|
+
};
|
|
165
|
+
if (extra && typeof extra === 'object' && !Array.isArray(extra)) {
|
|
166
|
+
Object.assign(payload, extra);
|
|
167
|
+
}
|
|
168
|
+
const body = JSON.stringify(payload);
|
|
101
169
|
res.writeHead(statusCode, {
|
|
102
170
|
'Content-Type': 'application/json',
|
|
103
171
|
'Access-Control-Allow-Origin': '*',
|
|
@@ -114,6 +182,10 @@ function jsonResponse(res, statusCode, data) {
|
|
|
114
182
|
res.end(body);
|
|
115
183
|
}
|
|
116
184
|
|
|
185
|
+
function requestError(message, statusCode = 400, type = 'invalid_request_error') {
|
|
186
|
+
return new ServeRequestError(message, statusCode, type);
|
|
187
|
+
}
|
|
188
|
+
|
|
117
189
|
async function readRequestBody(req) {
|
|
118
190
|
const chunks = [];
|
|
119
191
|
for await (const chunk of req) {
|
|
@@ -121,29 +193,29 @@ async function readRequestBody(req) {
|
|
|
121
193
|
}
|
|
122
194
|
const raw = Buffer.concat(chunks).toString('utf8');
|
|
123
195
|
if (!raw || raw.trim().length === 0) {
|
|
124
|
-
throw
|
|
196
|
+
throw requestError('Request body is empty.');
|
|
125
197
|
}
|
|
126
198
|
try {
|
|
127
199
|
return JSON.parse(raw);
|
|
128
200
|
} catch {
|
|
129
|
-
throw
|
|
201
|
+
throw requestError('Request body is not valid JSON.');
|
|
130
202
|
}
|
|
131
203
|
}
|
|
132
204
|
|
|
133
205
|
function validateMessages(messages) {
|
|
134
206
|
if (!Array.isArray(messages) || messages.length === 0) {
|
|
135
|
-
throw
|
|
207
|
+
throw requestError('"messages" must be a non-empty array.');
|
|
136
208
|
}
|
|
137
209
|
for (let i = 0; i < messages.length; i += 1) {
|
|
138
210
|
const msg = messages[i];
|
|
139
211
|
if (!msg || typeof msg !== 'object') {
|
|
140
|
-
throw
|
|
212
|
+
throw requestError(`messages[${i}] must be an object.`);
|
|
141
213
|
}
|
|
142
214
|
if (typeof msg.role !== 'string' || msg.role.trim().length === 0) {
|
|
143
|
-
throw
|
|
215
|
+
throw requestError(`messages[${i}].role must be a non-empty string.`);
|
|
144
216
|
}
|
|
145
217
|
if (typeof msg.content !== 'string') {
|
|
146
|
-
throw
|
|
218
|
+
throw requestError(`messages[${i}].content must be a string.`);
|
|
147
219
|
}
|
|
148
220
|
}
|
|
149
221
|
return messages.map((msg) => ({ role: msg.role.trim(), content: msg.content }));
|
|
@@ -153,36 +225,279 @@ function extractGenerationOptions(body) {
|
|
|
153
225
|
const options = {};
|
|
154
226
|
if (body.max_tokens != null) {
|
|
155
227
|
const n = Number(body.max_tokens);
|
|
156
|
-
if (!Number.isFinite(n) || n < 1) {
|
|
157
|
-
throw
|
|
228
|
+
if (!Number.isFinite(n) || n < 1 || Math.floor(n) !== n) {
|
|
229
|
+
throw requestError('"max_tokens" must be a positive integer.');
|
|
158
230
|
}
|
|
159
|
-
options.maxTokens =
|
|
231
|
+
options.maxTokens = n;
|
|
160
232
|
}
|
|
161
233
|
if (body.temperature != null) {
|
|
162
234
|
const t = Number(body.temperature);
|
|
163
235
|
if (!Number.isFinite(t) || t < 0) {
|
|
164
|
-
throw
|
|
236
|
+
throw requestError('"temperature" must be a non-negative number.');
|
|
165
237
|
}
|
|
166
238
|
options.temperature = t;
|
|
167
239
|
}
|
|
168
240
|
if (body.top_p != null) {
|
|
169
241
|
const p = Number(body.top_p);
|
|
170
242
|
if (!Number.isFinite(p) || p <= 0 || p > 1) {
|
|
171
|
-
throw
|
|
243
|
+
throw requestError('"top_p" must be a number between 0 (exclusive) and 1 (inclusive).');
|
|
172
244
|
}
|
|
173
245
|
options.topP = p;
|
|
174
246
|
}
|
|
175
247
|
if (body.top_k != null) {
|
|
176
248
|
const k = Number(body.top_k);
|
|
177
|
-
if (!Number.isFinite(k) || k < 1) {
|
|
178
|
-
throw
|
|
249
|
+
if (!Number.isFinite(k) || k < 1 || Math.floor(k) !== k) {
|
|
250
|
+
throw requestError('"top_k" must be a positive integer.');
|
|
179
251
|
}
|
|
180
|
-
options.topK =
|
|
252
|
+
options.topK = k;
|
|
181
253
|
}
|
|
182
254
|
return options;
|
|
183
255
|
}
|
|
184
256
|
|
|
185
|
-
|
|
257
|
+
function resolveIncludeReceipt(body) {
|
|
258
|
+
const includeReceipt = body.include_receipt;
|
|
259
|
+
const dopplerReceipt = body.doppler_receipt;
|
|
260
|
+
if (includeReceipt == null && dopplerReceipt == null) {
|
|
261
|
+
return false;
|
|
262
|
+
}
|
|
263
|
+
if (includeReceipt != null && typeof includeReceipt !== 'boolean') {
|
|
264
|
+
throw requestError('"include_receipt" must be a boolean when provided.');
|
|
265
|
+
}
|
|
266
|
+
if (dopplerReceipt != null && typeof dopplerReceipt !== 'boolean') {
|
|
267
|
+
throw requestError('"doppler_receipt" must be a boolean when provided.');
|
|
268
|
+
}
|
|
269
|
+
if (includeReceipt != null && dopplerReceipt != null && includeReceipt !== dopplerReceipt) {
|
|
270
|
+
throw requestError('"include_receipt" and "doppler_receipt" must agree when both are provided.');
|
|
271
|
+
}
|
|
272
|
+
return includeReceipt ?? dopplerReceipt;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
async function resolveTextModelEntry(modelId, resolveModel) {
|
|
276
|
+
let entry;
|
|
277
|
+
try {
|
|
278
|
+
entry = await resolveModel(modelId);
|
|
279
|
+
} catch {
|
|
280
|
+
throw requestError(`Unknown model "${modelId}". Use GET /v1/models for supported chat models.`);
|
|
281
|
+
}
|
|
282
|
+
if (!entry.modes.includes('text')) {
|
|
283
|
+
throw requestError(`Model "${modelId}" is not text-generative. Use GET /v1/models for supported chat models.`);
|
|
284
|
+
}
|
|
285
|
+
return entry;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function buildGenerationOptionsReceipt(generationOptions) {
|
|
289
|
+
return {
|
|
290
|
+
maxTokens: generationOptions.maxTokens ?? null,
|
|
291
|
+
temperature: generationOptions.temperature ?? null,
|
|
292
|
+
topP: generationOptions.topP ?? null,
|
|
293
|
+
topK: generationOptions.topK ?? null,
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
function normalizeRuntimeModelSource(runtimeModel, registryEntry) {
|
|
298
|
+
if (typeof runtimeModel === 'string') {
|
|
299
|
+
return {
|
|
300
|
+
kind: 'quickstart-registry',
|
|
301
|
+
modelId: runtimeModel,
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
if (runtimeModel && typeof runtimeModel === 'object' && typeof runtimeModel.url === 'string') {
|
|
305
|
+
return {
|
|
306
|
+
kind: 'url',
|
|
307
|
+
url: runtimeModel.url,
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
if (runtimeModel && typeof runtimeModel === 'object' && runtimeModel.manifest && typeof runtimeModel.manifest === 'object') {
|
|
311
|
+
return {
|
|
312
|
+
kind: 'inline-manifest',
|
|
313
|
+
modelId: typeof runtimeModel.manifest.modelId === 'string'
|
|
314
|
+
? runtimeModel.manifest.modelId
|
|
315
|
+
: registryEntry.modelId,
|
|
316
|
+
baseUrl: typeof runtimeModel.baseUrl === 'string' ? runtimeModel.baseUrl : null,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
return {
|
|
320
|
+
kind: 'quickstart-registry',
|
|
321
|
+
modelId: registryEntry.modelId,
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function buildServeReceiptBase({ requestedModel, registryEntry, messages, generationOptions, runtimeModel = null }) {
|
|
326
|
+
const generation = buildGenerationOptionsReceipt(generationOptions);
|
|
327
|
+
return {
|
|
328
|
+
receiptVersion: 'doppler_serve_receipt_v1',
|
|
329
|
+
schemaVersion: 1,
|
|
330
|
+
surface: 'serve',
|
|
331
|
+
endpoint: '/v1/chat/completions',
|
|
332
|
+
runtime: 'doppler-gpu',
|
|
333
|
+
runtimeVersion: DOPPLER_VERSION,
|
|
334
|
+
runtimePath: 'doppler-gpu.chatText',
|
|
335
|
+
runtimeModelSource: normalizeRuntimeModelSource(runtimeModel ?? registryEntry.modelId, registryEntry),
|
|
336
|
+
modelId: registryEntry.modelId,
|
|
337
|
+
requestedModel,
|
|
338
|
+
resolvedModel: registryEntry.modelId,
|
|
339
|
+
artifact: {
|
|
340
|
+
format: 'rdrr',
|
|
341
|
+
source: 'quickstart-registry',
|
|
342
|
+
sourceCheckpointId: registryEntry.sourceCheckpointId,
|
|
343
|
+
weightPackId: registryEntry.weightPackId,
|
|
344
|
+
manifestVariantId: registryEntry.manifestVariantId,
|
|
345
|
+
artifactCompleteness: registryEntry.artifactCompleteness,
|
|
346
|
+
runtimePromotionState: registryEntry.runtimePromotionState,
|
|
347
|
+
weightsRefAllowed: registryEntry.weightsRefAllowed,
|
|
348
|
+
hf: registryEntry.hf,
|
|
349
|
+
},
|
|
350
|
+
request: {
|
|
351
|
+
messages: {
|
|
352
|
+
count: messages.length,
|
|
353
|
+
digest: buildJsonSha256Evidence(messages),
|
|
354
|
+
},
|
|
355
|
+
generationDigest: buildJsonSha256Evidence(generation),
|
|
356
|
+
},
|
|
357
|
+
generation,
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
export function buildServeReceipt({
|
|
362
|
+
requestedModel,
|
|
363
|
+
registryEntry,
|
|
364
|
+
messages,
|
|
365
|
+
generationOptions,
|
|
366
|
+
outputContent,
|
|
367
|
+
usage,
|
|
368
|
+
runtimeModel = null,
|
|
369
|
+
}) {
|
|
370
|
+
const baseReceipt = buildServeReceiptBase({
|
|
371
|
+
requestedModel,
|
|
372
|
+
registryEntry,
|
|
373
|
+
messages,
|
|
374
|
+
generationOptions,
|
|
375
|
+
runtimeModel,
|
|
376
|
+
});
|
|
377
|
+
const outputText = String(outputContent ?? '');
|
|
378
|
+
return {
|
|
379
|
+
...baseReceipt,
|
|
380
|
+
status: 'pass',
|
|
381
|
+
output: {
|
|
382
|
+
role: 'assistant',
|
|
383
|
+
digest: buildSha256Evidence(outputText),
|
|
384
|
+
textLength: outputText.length,
|
|
385
|
+
empty: outputText.length === 0,
|
|
386
|
+
},
|
|
387
|
+
transcript: {
|
|
388
|
+
digest: buildJsonSha256Evidence({
|
|
389
|
+
messages,
|
|
390
|
+
generation: baseReceipt.generation,
|
|
391
|
+
output: outputText,
|
|
392
|
+
usage,
|
|
393
|
+
}),
|
|
394
|
+
},
|
|
395
|
+
usage,
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
function normalizeWeightLoadFailure(error) {
|
|
400
|
+
const failure = error?.details?.weightLoadFailure ?? error?.cause?.details?.weightLoadFailure;
|
|
401
|
+
if (!failure || typeof failure !== 'object') {
|
|
402
|
+
return null;
|
|
403
|
+
}
|
|
404
|
+
const deviceLimitFailure = failure.deviceLimitFailure && typeof failure.deviceLimitFailure === 'object'
|
|
405
|
+
? {
|
|
406
|
+
kind: typeof failure.deviceLimitFailure.kind === 'string' ? failure.deviceLimitFailure.kind : null,
|
|
407
|
+
maxGpuResidentBytes: Number.isFinite(failure.deviceLimitFailure.maxGpuResidentBytes)
|
|
408
|
+
? failure.deviceLimitFailure.maxGpuResidentBytes
|
|
409
|
+
: null,
|
|
410
|
+
maxStorageBufferBindingSize: Number.isFinite(failure.deviceLimitFailure.maxStorageBufferBindingSize)
|
|
411
|
+
? failure.deviceLimitFailure.maxStorageBufferBindingSize
|
|
412
|
+
: null,
|
|
413
|
+
maxBufferSize: Number.isFinite(failure.deviceLimitFailure.maxBufferSize)
|
|
414
|
+
? failure.deviceLimitFailure.maxBufferSize
|
|
415
|
+
: null,
|
|
416
|
+
maxStorageBuffersPerShaderStage: Number.isFinite(failure.deviceLimitFailure.maxStorageBuffersPerShaderStage)
|
|
417
|
+
? failure.deviceLimitFailure.maxStorageBuffersPerShaderStage
|
|
418
|
+
: null,
|
|
419
|
+
largeWeightMaxBytes: Number.isFinite(failure.deviceLimitFailure.largeWeightMaxBytes)
|
|
420
|
+
? failure.deviceLimitFailure.largeWeightMaxBytes
|
|
421
|
+
: null,
|
|
422
|
+
embeddingKernel: failure.deviceLimitFailure.embeddingKernel
|
|
423
|
+
&& typeof failure.deviceLimitFailure.embeddingKernel === 'object'
|
|
424
|
+
? {
|
|
425
|
+
kernel: typeof failure.deviceLimitFailure.embeddingKernel.kernel === 'string'
|
|
426
|
+
? failure.deviceLimitFailure.embeddingKernel.kernel
|
|
427
|
+
: null,
|
|
428
|
+
entry: typeof failure.deviceLimitFailure.embeddingKernel.entry === 'string'
|
|
429
|
+
? failure.deviceLimitFailure.embeddingKernel.entry
|
|
430
|
+
: null,
|
|
431
|
+
}
|
|
432
|
+
: null,
|
|
433
|
+
splitKernelExpected: typeof failure.deviceLimitFailure.splitKernelExpected === 'boolean'
|
|
434
|
+
? failure.deviceLimitFailure.splitKernelExpected
|
|
435
|
+
: null,
|
|
436
|
+
activeSplitKernelMaxSections: Number.isFinite(failure.deviceLimitFailure.activeSplitKernelMaxSections)
|
|
437
|
+
? failure.deviceLimitFailure.activeSplitKernelMaxSections
|
|
438
|
+
: null,
|
|
439
|
+
maxSplitEmbeddingSections: Number.isFinite(failure.deviceLimitFailure.maxSplitEmbeddingSections)
|
|
440
|
+
? failure.deviceLimitFailure.maxSplitEmbeddingSections
|
|
441
|
+
: null,
|
|
442
|
+
requiredSplitSections: Number.isFinite(failure.deviceLimitFailure.requiredSplitSections)
|
|
443
|
+
? failure.deviceLimitFailure.requiredSplitSections
|
|
444
|
+
: null,
|
|
445
|
+
}
|
|
446
|
+
: null;
|
|
447
|
+
return {
|
|
448
|
+
tensorName: typeof failure.tensorName === 'string' ? failure.tensorName : null,
|
|
449
|
+
tensorRole: typeof failure.tensorRole === 'string' ? failure.tensorRole : null,
|
|
450
|
+
tensorDtype: typeof failure.tensorDtype === 'string' ? failure.tensorDtype : null,
|
|
451
|
+
tensorShape: Array.isArray(failure.tensorShape) ? [...failure.tensorShape] : null,
|
|
452
|
+
tensorSizeBytes: Number.isFinite(failure.tensorSizeBytes) ? failure.tensorSizeBytes : null,
|
|
453
|
+
tensorLoadStage: typeof failure.tensorLoadStage === 'string' ? failure.tensorLoadStage : null,
|
|
454
|
+
toGPU: typeof failure.toGPU === 'boolean' ? failure.toGPU : null,
|
|
455
|
+
streamedUpload: typeof failure.streamedUpload === 'boolean' ? failure.streamedUpload : null,
|
|
456
|
+
deviceLimitFailure,
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function normalizeServeFailure(error, registryEntry) {
|
|
461
|
+
const message = error?.message || String(error);
|
|
462
|
+
const pipelineLoadPhase = typeof error?.details?.pipelineLoadPhase === 'string'
|
|
463
|
+
? error.details.pipelineLoadPhase
|
|
464
|
+
: null;
|
|
465
|
+
return {
|
|
466
|
+
code: pipelineLoadPhase ? 'pipeline-load-failed' : 'runtime-error',
|
|
467
|
+
stage: pipelineLoadPhase ?? 'runtime',
|
|
468
|
+
message,
|
|
469
|
+
modelId: typeof error?.details?.modelId === 'string' ? error.details.modelId : registryEntry.modelId,
|
|
470
|
+
weightLoadFailure: normalizeWeightLoadFailure(error),
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
export function buildServeFailureReceipt({
|
|
475
|
+
requestedModel,
|
|
476
|
+
registryEntry,
|
|
477
|
+
messages,
|
|
478
|
+
generationOptions,
|
|
479
|
+
error,
|
|
480
|
+
runtimeModel = null,
|
|
481
|
+
}) {
|
|
482
|
+
return {
|
|
483
|
+
...buildServeReceiptBase({ requestedModel, registryEntry, messages, generationOptions, runtimeModel }),
|
|
484
|
+
status: 'diagnostic',
|
|
485
|
+
failure: normalizeServeFailure(error, registryEntry),
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
function resolveServeDependencies(dependencies = {}) {
|
|
490
|
+
return {
|
|
491
|
+
dopplerClient: dependencies.dopplerClient ?? doppler,
|
|
492
|
+
listModels: dependencies.listModels ?? listQuickstartModels,
|
|
493
|
+
resolveModel: dependencies.resolveModel ?? resolveQuickstartModel,
|
|
494
|
+
resolveRuntimeModel: typeof dependencies.resolveRuntimeModel === 'function'
|
|
495
|
+
? dependencies.resolveRuntimeModel
|
|
496
|
+
: (registryEntry) => registryEntry.modelId,
|
|
497
|
+
};
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
async function handleChatCompletions(req, res, dependencies) {
|
|
186
501
|
const body = await readRequestBody(req);
|
|
187
502
|
if (typeof body.model !== 'string' || body.model.trim().length === 0) {
|
|
188
503
|
return jsonError(res, 400, '"model" is required and must be a non-empty string.');
|
|
@@ -190,23 +505,81 @@ async function handleChatCompletions(req, res) {
|
|
|
190
505
|
const modelId = body.model.trim();
|
|
191
506
|
const messages = validateMessages(body.messages);
|
|
192
507
|
const generationOptions = extractGenerationOptions(body);
|
|
508
|
+
const includeReceipt = resolveIncludeReceipt(body);
|
|
193
509
|
const stream = body.stream === true;
|
|
510
|
+
if (includeReceipt && stream) {
|
|
511
|
+
throw requestError('"include_receipt" is not supported with streaming responses.');
|
|
512
|
+
}
|
|
513
|
+
const registryEntry = await resolveTextModelEntry(modelId, dependencies.resolveModel);
|
|
194
514
|
const completionId = generateCompletionId();
|
|
195
515
|
const created = Math.floor(Date.now() / 1000);
|
|
196
516
|
|
|
197
517
|
if (stream) {
|
|
198
|
-
return handleStreamingCompletion(
|
|
518
|
+
return handleStreamingCompletion(
|
|
519
|
+
res,
|
|
520
|
+
modelId,
|
|
521
|
+
registryEntry,
|
|
522
|
+
messages,
|
|
523
|
+
generationOptions,
|
|
524
|
+
completionId,
|
|
525
|
+
created,
|
|
526
|
+
dependencies
|
|
527
|
+
);
|
|
199
528
|
}
|
|
200
|
-
return handleNonStreamingCompletion(
|
|
529
|
+
return handleNonStreamingCompletion(
|
|
530
|
+
res,
|
|
531
|
+
modelId,
|
|
532
|
+
registryEntry,
|
|
533
|
+
messages,
|
|
534
|
+
generationOptions,
|
|
535
|
+
completionId,
|
|
536
|
+
created,
|
|
537
|
+
includeReceipt,
|
|
538
|
+
dependencies
|
|
539
|
+
);
|
|
201
540
|
}
|
|
202
541
|
|
|
203
|
-
async function handleNonStreamingCompletion(
|
|
204
|
-
|
|
205
|
-
|
|
542
|
+
async function handleNonStreamingCompletion(
|
|
543
|
+
res,
|
|
544
|
+
requestedModel,
|
|
545
|
+
registryEntry,
|
|
546
|
+
messages,
|
|
547
|
+
generationOptions,
|
|
548
|
+
completionId,
|
|
549
|
+
created,
|
|
550
|
+
includeReceipt,
|
|
551
|
+
dependencies
|
|
552
|
+
) {
|
|
553
|
+
let result;
|
|
554
|
+
const runtimeModel = dependencies.resolveRuntimeModel(registryEntry, requestedModel);
|
|
555
|
+
try {
|
|
556
|
+
result = await dependencies.dopplerClient.chatText(messages, { model: runtimeModel, ...generationOptions });
|
|
557
|
+
} catch (error) {
|
|
558
|
+
if (includeReceipt) {
|
|
559
|
+
return jsonError(
|
|
560
|
+
res,
|
|
561
|
+
500,
|
|
562
|
+
error?.message || String(error),
|
|
563
|
+
'server_error',
|
|
564
|
+
{
|
|
565
|
+
doppler_receipt: buildServeFailureReceipt({
|
|
566
|
+
requestedModel,
|
|
567
|
+
registryEntry,
|
|
568
|
+
messages,
|
|
569
|
+
generationOptions,
|
|
570
|
+
error,
|
|
571
|
+
runtimeModel,
|
|
572
|
+
}),
|
|
573
|
+
}
|
|
574
|
+
);
|
|
575
|
+
}
|
|
576
|
+
throw error;
|
|
577
|
+
}
|
|
578
|
+
const body = {
|
|
206
579
|
id: completionId,
|
|
207
580
|
object: 'chat.completion',
|
|
208
581
|
created,
|
|
209
|
-
model: modelId,
|
|
582
|
+
model: registryEntry.modelId,
|
|
210
583
|
choices: [
|
|
211
584
|
{
|
|
212
585
|
index: 0,
|
|
@@ -222,10 +595,32 @@ async function handleNonStreamingCompletion(res, modelId, messages, generationOp
|
|
|
222
595
|
completion_tokens: result.usage.completionTokens,
|
|
223
596
|
total_tokens: result.usage.totalTokens,
|
|
224
597
|
},
|
|
225
|
-
}
|
|
598
|
+
};
|
|
599
|
+
if (includeReceipt) {
|
|
600
|
+
body.doppler_receipt = buildServeReceipt({
|
|
601
|
+
requestedModel,
|
|
602
|
+
registryEntry,
|
|
603
|
+
messages,
|
|
604
|
+
generationOptions,
|
|
605
|
+
outputContent: result.content,
|
|
606
|
+
usage: result.usage,
|
|
607
|
+
runtimeModel,
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
jsonResponse(res, 200, body);
|
|
226
611
|
}
|
|
227
612
|
|
|
228
|
-
async function handleStreamingCompletion(
|
|
613
|
+
async function handleStreamingCompletion(
|
|
614
|
+
res,
|
|
615
|
+
requestedModel,
|
|
616
|
+
registryEntry,
|
|
617
|
+
messages,
|
|
618
|
+
generationOptions,
|
|
619
|
+
completionId,
|
|
620
|
+
created,
|
|
621
|
+
dependencies
|
|
622
|
+
) {
|
|
623
|
+
const runtimeModel = dependencies.resolveRuntimeModel(registryEntry, requestedModel);
|
|
229
624
|
res.writeHead(200, {
|
|
230
625
|
'Content-Type': 'text/event-stream',
|
|
231
626
|
'Cache-Control': 'no-cache',
|
|
@@ -241,7 +636,7 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
|
|
|
241
636
|
id: completionId,
|
|
242
637
|
object: 'chat.completion.chunk',
|
|
243
638
|
created,
|
|
244
|
-
model: modelId,
|
|
639
|
+
model: registryEntry.modelId,
|
|
245
640
|
choices: [
|
|
246
641
|
{
|
|
247
642
|
index: 0,
|
|
@@ -251,7 +646,7 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
|
|
|
251
646
|
],
|
|
252
647
|
});
|
|
253
648
|
|
|
254
|
-
const stream =
|
|
649
|
+
const stream = dependencies.dopplerClient.chat(messages, { model: runtimeModel, ...generationOptions });
|
|
255
650
|
for await (const token of stream) {
|
|
256
651
|
if (res.destroyed) {
|
|
257
652
|
break;
|
|
@@ -260,7 +655,7 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
|
|
|
260
655
|
id: completionId,
|
|
261
656
|
object: 'chat.completion.chunk',
|
|
262
657
|
created,
|
|
263
|
-
model: modelId,
|
|
658
|
+
model: registryEntry.modelId,
|
|
264
659
|
choices: [
|
|
265
660
|
{
|
|
266
661
|
index: 0,
|
|
@@ -275,7 +670,7 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
|
|
|
275
670
|
id: completionId,
|
|
276
671
|
object: 'chat.completion.chunk',
|
|
277
672
|
created,
|
|
278
|
-
model: modelId,
|
|
673
|
+
model: registryEntry.modelId,
|
|
279
674
|
choices: [
|
|
280
675
|
{
|
|
281
676
|
index: 0,
|
|
@@ -288,8 +683,8 @@ async function handleStreamingCompletion(res, modelId, messages, generationOptio
|
|
|
288
683
|
res.end();
|
|
289
684
|
}
|
|
290
685
|
|
|
291
|
-
async function handleListModels(res) {
|
|
292
|
-
const models = await
|
|
686
|
+
async function handleListModels(res, dependencies) {
|
|
687
|
+
const models = await dependencies.listModels();
|
|
293
688
|
const textModels = models.filter((entry) => entry.modes.includes('text'));
|
|
294
689
|
jsonResponse(res, 200, {
|
|
295
690
|
object: 'list',
|
|
@@ -298,6 +693,15 @@ async function handleListModels(res) {
|
|
|
298
693
|
object: 'model',
|
|
299
694
|
created: 0,
|
|
300
695
|
owned_by: 'doppler',
|
|
696
|
+
doppler: {
|
|
697
|
+
sourceCheckpointId: entry.sourceCheckpointId,
|
|
698
|
+
weightPackId: entry.weightPackId,
|
|
699
|
+
manifestVariantId: entry.manifestVariantId,
|
|
700
|
+
artifactCompleteness: entry.artifactCompleteness,
|
|
701
|
+
runtimePromotionState: entry.runtimePromotionState,
|
|
702
|
+
weightsRefAllowed: entry.weightsRefAllowed,
|
|
703
|
+
modes: entry.modes,
|
|
704
|
+
},
|
|
301
705
|
})),
|
|
302
706
|
});
|
|
303
707
|
}
|
|
@@ -319,7 +723,8 @@ function handleCors(req, res) {
|
|
|
319
723
|
res.end();
|
|
320
724
|
}
|
|
321
725
|
|
|
322
|
-
export function createServeHandler() {
|
|
726
|
+
export function createServeHandler(dependencies = {}) {
|
|
727
|
+
const resolvedDependencies = resolveServeDependencies(dependencies);
|
|
323
728
|
return async (req, res) => {
|
|
324
729
|
if (req.method === 'OPTIONS') {
|
|
325
730
|
return handleCors(req, res);
|
|
@@ -330,10 +735,10 @@ export function createServeHandler() {
|
|
|
330
735
|
|
|
331
736
|
try {
|
|
332
737
|
if (pathname === '/v1/chat/completions' && req.method === 'POST') {
|
|
333
|
-
return await handleChatCompletions(req, res);
|
|
738
|
+
return await handleChatCompletions(req, res, resolvedDependencies);
|
|
334
739
|
}
|
|
335
740
|
if (pathname === '/v1/models' && req.method === 'GET') {
|
|
336
|
-
return await handleListModels(res);
|
|
741
|
+
return await handleListModels(res, resolvedDependencies);
|
|
337
742
|
}
|
|
338
743
|
if ((pathname === '/health' || pathname === '/') && req.method === 'GET') {
|
|
339
744
|
return handleHealth(res);
|
|
@@ -343,20 +748,41 @@ export function createServeHandler() {
|
|
|
343
748
|
const message = error?.message || String(error);
|
|
344
749
|
console.error(`[doppler-serve] ${req.method} ${pathname}: ${message}`);
|
|
345
750
|
if (!res.headersSent) {
|
|
346
|
-
|
|
751
|
+
if (error instanceof ServeRequestError) {
|
|
752
|
+
jsonError(res, error.statusCode, message, error.type);
|
|
753
|
+
} else {
|
|
754
|
+
jsonError(res, 500, message, 'server_error');
|
|
755
|
+
}
|
|
347
756
|
}
|
|
348
757
|
}
|
|
349
758
|
};
|
|
350
759
|
}
|
|
351
760
|
|
|
761
|
+
function createRuntimeModelResolver(localModelSourceByModelId = new Map()) {
|
|
762
|
+
return (registryEntry) => localModelSourceByModelId.get(registryEntry.modelId) ?? registryEntry.modelId;
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
async function resolveLocalModelSourceMap(settings) {
|
|
766
|
+
const localModelSourceByModelId = new Map();
|
|
767
|
+
if (!settings.modelUrl) {
|
|
768
|
+
return localModelSourceByModelId;
|
|
769
|
+
}
|
|
770
|
+
const registryEntry = await resolveQuickstartModel(settings.model);
|
|
771
|
+
localModelSourceByModelId.set(registryEntry.modelId, { url: settings.modelUrl });
|
|
772
|
+
return localModelSourceByModelId;
|
|
773
|
+
}
|
|
774
|
+
|
|
352
775
|
async function startServer(settings) {
|
|
776
|
+
const localModelSourceByModelId = await resolveLocalModelSourceMap(settings);
|
|
777
|
+
const resolveRuntimeModel = createRuntimeModelResolver(localModelSourceByModelId);
|
|
353
778
|
if (settings.model) {
|
|
354
779
|
console.error(`[doppler-serve] pre-loading model: ${settings.model}`);
|
|
355
|
-
await
|
|
780
|
+
const registryEntry = await resolveQuickstartModel(settings.model);
|
|
781
|
+
await doppler.load(resolveRuntimeModel(registryEntry));
|
|
356
782
|
console.error(`[doppler-serve] model ready: ${settings.model}`);
|
|
357
783
|
}
|
|
358
784
|
|
|
359
|
-
const handler = createServeHandler();
|
|
785
|
+
const handler = createServeHandler({ resolveRuntimeModel });
|
|
360
786
|
const server = http.createServer(handler);
|
|
361
787
|
|
|
362
788
|
return new Promise((resolve, reject) => {
|