doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
|
|
2
|
-
import { join, resolve } from 'node:path';
|
|
1
|
+
import { access, mkdir, readFile, readdir, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname, isAbsolute, join, resolve } from 'node:path';
|
|
3
|
+
import { pathToFileURL } from 'node:url';
|
|
3
4
|
|
|
4
5
|
import { loadBackwardRegistry } from '../../config/backward-registry-loader.js';
|
|
5
6
|
import { acquireBuffer, readBuffer, releaseBuffer, uploadData } from '../../memory/buffer-pool.js';
|
|
6
7
|
import { runMatmul } from '../../gpu/kernels/index.js';
|
|
7
8
|
import { runResidualAdd } from '../../gpu/kernels/residual.js';
|
|
8
9
|
import { parseJsonl } from './datasets/jsonl.js';
|
|
10
|
+
import { loadTextPairsDataset, tokenizeTextPairs } from './datasets/text-pairs.js';
|
|
9
11
|
import { LoraAdapter } from './lora.js';
|
|
10
12
|
import { TrainingRunner, restoreTrainingCheckpointState } from './runner.js';
|
|
11
13
|
import { AdamOptimizer } from './optimizer.js';
|
|
@@ -15,6 +17,7 @@ import { OpType, AutogradTape } from './autograd.js';
|
|
|
15
17
|
import { loadCheckpoint } from './checkpoint.js';
|
|
16
18
|
import { exportLoRAAdapter } from './export.js';
|
|
17
19
|
import { computeEvalMetrics } from './operator-eval.js';
|
|
20
|
+
import { summarizeAgentEvalReportRequirements } from './operator-agent-eval.js';
|
|
18
21
|
import { appendScoreboardRow } from './operator-scoreboard.js';
|
|
19
22
|
import {
|
|
20
23
|
buildArtifactBase,
|
|
@@ -28,6 +31,331 @@ import { watchFinalizedCheckpoints } from './checkpoint-watch.js';
|
|
|
28
31
|
import { loadLoRAFromManifest } from '../adapters/lora-loader.js';
|
|
29
32
|
import { createUploadedTensor } from './tensor-factory.js';
|
|
30
33
|
import { stableSortObject } from '../../utils/stable-sort-object.js';
|
|
34
|
+
import { LORA_MODULE_ALIASES } from '../../inference/pipelines/text/lora.js';
|
|
35
|
+
import { loadDistillModelHandle } from './suite.js';
|
|
36
|
+
import { createDistillStudentRuntimeModelFixture } from './distillation/student-fixture.js';
|
|
37
|
+
import { f16ToF32Array } from '../../inference/kv-cache/types.js';
|
|
38
|
+
|
|
39
|
+
const CAUSAL_LM_TEXT_PAIR_RUNNER_KEYS = Object.freeze([
|
|
40
|
+
'gemma-3-270m-it-f16-af32::text-pairs::text_generation',
|
|
41
|
+
'gemma-3-270m-it-q4k-ehf16-af32::text-pairs::text_generation',
|
|
42
|
+
'gemma4-e2b-it::text-pairs::text_generation',
|
|
43
|
+
'gemma-4-e2b-it-q4k-ehf16-af32::text-pairs::text_generation',
|
|
44
|
+
'gemma-4-e2b-it-q4k-ehf16-af32-int4ple::text-pairs::text_generation',
|
|
45
|
+
'qwen-3-5-0-8b-q4k-ehaf16::text-pairs::text_generation',
|
|
46
|
+
'qwen-3-5-2b-q4k-ehaf16::text-pairs::text_generation',
|
|
47
|
+
'qwen-3-6-27b-q4k-ehaf16::text-pairs::text_generation',
|
|
48
|
+
'qwen-3-6-27b-q4k-eaf16::text-pairs::text_generation',
|
|
49
|
+
]);
|
|
50
|
+
|
|
51
|
+
export const LORA_RUNNER_SUPPORT_CONTRACT = Object.freeze({
|
|
52
|
+
supportedBaseModelId: 'training-toy',
|
|
53
|
+
supportedDatasetFormat: 'toy_linear_classification_jsonl',
|
|
54
|
+
registeredBaseModelIds: Object.freeze([
|
|
55
|
+
'training-toy',
|
|
56
|
+
'gemma-3-270m-it-f16-af32',
|
|
57
|
+
'gemma-3-270m-it-q4k-ehf16-af32',
|
|
58
|
+
'gemma4-e2b-it',
|
|
59
|
+
'gemma-4-e2b-it-q4k-ehf16-af32',
|
|
60
|
+
'gemma-4-e2b-it-q4k-ehf16-af32-int4ple',
|
|
61
|
+
'qwen-3-5-0-8b-q4k-ehaf16',
|
|
62
|
+
'qwen-3-5-2b-q4k-ehaf16',
|
|
63
|
+
'qwen-3-6-27b-q4k-ehaf16',
|
|
64
|
+
'qwen-3-6-27b-q4k-eaf16',
|
|
65
|
+
]),
|
|
66
|
+
registeredDatasetFormats: Object.freeze([
|
|
67
|
+
'toy_linear_classification_jsonl',
|
|
68
|
+
'text-pairs',
|
|
69
|
+
]),
|
|
70
|
+
implementedRunnerKeys: Object.freeze([
|
|
71
|
+
'training-toy::toy_linear_classification_jsonl::classification',
|
|
72
|
+
...CAUSAL_LM_TEXT_PAIR_RUNNER_KEYS,
|
|
73
|
+
]),
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
export const LORA_RUNNER_BASE_MODEL_REGISTRY = Object.freeze({
|
|
77
|
+
'training-toy': Object.freeze({
|
|
78
|
+
baseModelId: 'training-toy',
|
|
79
|
+
family: 'training_fixture',
|
|
80
|
+
runnerKind: 'toy_linear_classification',
|
|
81
|
+
}),
|
|
82
|
+
'gemma-3-270m-it-f16-af32': Object.freeze({
|
|
83
|
+
baseModelId: 'gemma-3-270m-it-f16-af32',
|
|
84
|
+
modelRef: 'gemma-3-270m-it-f16-af32',
|
|
85
|
+
family: 'gemma3',
|
|
86
|
+
runnerKind: 'causal_lm_text_generation',
|
|
87
|
+
}),
|
|
88
|
+
'gemma-3-270m-it-q4k-ehf16-af32': Object.freeze({
|
|
89
|
+
baseModelId: 'gemma-3-270m-it-q4k-ehf16-af32',
|
|
90
|
+
modelRef: 'gemma-3-270m-it-q4k-ehf16-af32',
|
|
91
|
+
family: 'gemma3',
|
|
92
|
+
runnerKind: 'causal_lm_text_generation',
|
|
93
|
+
requiresExternalTrainer: true,
|
|
94
|
+
}),
|
|
95
|
+
'gemma4-e2b-it': Object.freeze({
|
|
96
|
+
baseModelId: 'gemma4-e2b-it',
|
|
97
|
+
modelRef: 'gemma-4-e2b-it-q4k-ehf16-af32',
|
|
98
|
+
family: 'gemma4',
|
|
99
|
+
runnerKind: 'causal_lm_text_generation',
|
|
100
|
+
requiresExternalTrainer: true,
|
|
101
|
+
}),
|
|
102
|
+
'gemma-4-e2b-it-q4k-ehf16-af32': Object.freeze({
|
|
103
|
+
baseModelId: 'gemma-4-e2b-it-q4k-ehf16-af32',
|
|
104
|
+
modelRef: 'gemma-4-e2b-it-q4k-ehf16-af32',
|
|
105
|
+
family: 'gemma4',
|
|
106
|
+
runnerKind: 'causal_lm_text_generation',
|
|
107
|
+
requiresExternalTrainer: true,
|
|
108
|
+
}),
|
|
109
|
+
'gemma-4-e2b-it-q4k-ehf16-af32-int4ple': Object.freeze({
|
|
110
|
+
baseModelId: 'gemma-4-e2b-it-q4k-ehf16-af32-int4ple',
|
|
111
|
+
modelRef: 'gemma-4-e2b-it-q4k-ehf16-af32-int4ple',
|
|
112
|
+
family: 'gemma4',
|
|
113
|
+
runnerKind: 'causal_lm_text_generation',
|
|
114
|
+
requiresExternalTrainer: true,
|
|
115
|
+
}),
|
|
116
|
+
'qwen-3-5-0-8b-q4k-ehaf16': Object.freeze({
|
|
117
|
+
baseModelId: 'qwen-3-5-0-8b-q4k-ehaf16',
|
|
118
|
+
modelRef: 'qwen-3-5-0-8b-q4k-ehaf16',
|
|
119
|
+
family: 'qwen3',
|
|
120
|
+
runnerKind: 'causal_lm_text_generation',
|
|
121
|
+
requiresExternalTrainer: true,
|
|
122
|
+
}),
|
|
123
|
+
'qwen-3-5-2b-q4k-ehaf16': Object.freeze({
|
|
124
|
+
baseModelId: 'qwen-3-5-2b-q4k-ehaf16',
|
|
125
|
+
modelRef: 'qwen-3-5-2b-q4k-ehaf16',
|
|
126
|
+
family: 'qwen3',
|
|
127
|
+
runnerKind: 'causal_lm_text_generation',
|
|
128
|
+
requiresExternalTrainer: true,
|
|
129
|
+
}),
|
|
130
|
+
'qwen-3-6-27b-q4k-ehaf16': Object.freeze({
|
|
131
|
+
baseModelId: 'qwen-3-6-27b-q4k-ehaf16',
|
|
132
|
+
modelRef: 'qwen-3-6-27b-q4k-ehaf16',
|
|
133
|
+
family: 'qwen3',
|
|
134
|
+
runnerKind: 'causal_lm_text_generation',
|
|
135
|
+
requiresExternalTrainer: true,
|
|
136
|
+
}),
|
|
137
|
+
'qwen-3-6-27b-q4k-eaf16': Object.freeze({
|
|
138
|
+
baseModelId: 'qwen-3-6-27b-q4k-eaf16',
|
|
139
|
+
modelRef: 'qwen-3-6-27b-q4k-eaf16',
|
|
140
|
+
family: 'qwen3',
|
|
141
|
+
runnerKind: 'causal_lm_text_generation',
|
|
142
|
+
requiresExternalTrainer: true,
|
|
143
|
+
}),
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
export const LORA_RUNNER_DATASET_FORMAT_REGISTRY = Object.freeze({
|
|
147
|
+
toy_linear_classification_jsonl: Object.freeze({
|
|
148
|
+
datasetFormat: 'toy_linear_classification_jsonl',
|
|
149
|
+
datasetKind: 'toy_linear_classification',
|
|
150
|
+
}),
|
|
151
|
+
'text-pairs': Object.freeze({
|
|
152
|
+
datasetFormat: 'text-pairs',
|
|
153
|
+
datasetKind: 'causal_lm_text_pairs',
|
|
154
|
+
}),
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
function getPipelineConfig(workload) {
|
|
158
|
+
return workload?.pipeline || workload?.lora || {};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function normalizeLoraTargetModules(adapter) {
|
|
162
|
+
const rawModules = Array.isArray(adapter?.targetModules) ? adapter.targetModules : [];
|
|
163
|
+
const modules = [];
|
|
164
|
+
for (const rawModule of rawModules) {
|
|
165
|
+
const normalized = String(rawModule || '').trim();
|
|
166
|
+
if (!normalized) continue;
|
|
167
|
+
const moduleName = LORA_MODULE_ALIASES[normalized] || normalized;
|
|
168
|
+
if (!modules.includes(moduleName)) {
|
|
169
|
+
modules.push(moduleName);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
if (modules.length === 0) {
|
|
173
|
+
throw new Error('Causal-LM LoRA workload requires adapter.targetModules.');
|
|
174
|
+
}
|
|
175
|
+
return modules;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function getCausalLmBaseModelRef(workload) {
|
|
179
|
+
const pipeline = getPipelineConfig(workload);
|
|
180
|
+
const baseModel = LORA_RUNNER_BASE_MODEL_REGISTRY[String(workload?.baseModelId || '')] || null;
|
|
181
|
+
return String(
|
|
182
|
+
pipeline.baseModelRef
|
|
183
|
+
|| workload.baseModelRef
|
|
184
|
+
|| workload.studentModelId
|
|
185
|
+
|| baseModel?.modelRef
|
|
186
|
+
|| workload.baseModelId
|
|
187
|
+
|| ''
|
|
188
|
+
);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function getRunnerKey(baseModelId, datasetFormat, taskType) {
|
|
192
|
+
return `${baseModelId}::${datasetFormat}::${taskType}`;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function isCausalLmLoraWorkload(workload, compatibility = getLoraRunnerCompatibility(workload)) {
|
|
196
|
+
return compatibility.observed.baseModelRunnerKind === 'causal_lm_text_generation'
|
|
197
|
+
|| compatibility.observed.datasetKind === 'causal_lm_text_pairs'
|
|
198
|
+
|| compatibility.observed.taskType === 'text_generation';
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
export function getLoraRunnerCompatibility(workload) {
|
|
202
|
+
const baseModelId = String(workload?.baseModelId || '');
|
|
203
|
+
const pipeline = getPipelineConfig(workload);
|
|
204
|
+
const datasetFormat = String(pipeline?.datasetFormat || '');
|
|
205
|
+
const taskType = String(pipeline?.taskType || '');
|
|
206
|
+
const baseModel = LORA_RUNNER_BASE_MODEL_REGISTRY[baseModelId] || null;
|
|
207
|
+
const dataset = LORA_RUNNER_DATASET_FORMAT_REGISTRY[datasetFormat] || null;
|
|
208
|
+
const runnerKey = getRunnerKey(baseModelId, datasetFormat, taskType);
|
|
209
|
+
const blockedReasons = [];
|
|
210
|
+
if (!baseModel) {
|
|
211
|
+
blockedReasons.push('base_model_not_registered_for_current_lora_runner');
|
|
212
|
+
}
|
|
213
|
+
if (!dataset) {
|
|
214
|
+
blockedReasons.push('dataset_format_not_supported_by_current_lora_runner');
|
|
215
|
+
}
|
|
216
|
+
if (baseModel && dataset && !LORA_RUNNER_SUPPORT_CONTRACT.implementedRunnerKeys.includes(runnerKey)) {
|
|
217
|
+
blockedReasons.push('runner_combination_not_supported_by_current_lora_runner');
|
|
218
|
+
}
|
|
219
|
+
return {
|
|
220
|
+
schemaVersion: 1,
|
|
221
|
+
supported: blockedReasons.length === 0,
|
|
222
|
+
runnerContract: LORA_RUNNER_SUPPORT_CONTRACT,
|
|
223
|
+
observed: {
|
|
224
|
+
baseModelId,
|
|
225
|
+
datasetFormat,
|
|
226
|
+
taskType,
|
|
227
|
+
runnerKey,
|
|
228
|
+
baseModelFamily: baseModel?.family || null,
|
|
229
|
+
baseModelRunnerKind: baseModel?.runnerKind || null,
|
|
230
|
+
requiresExternalTrainer: baseModel?.requiresExternalTrainer === true,
|
|
231
|
+
datasetKind: dataset?.datasetKind || null,
|
|
232
|
+
registeredBaseModel: Boolean(baseModel),
|
|
233
|
+
registeredDatasetFormat: Boolean(dataset),
|
|
234
|
+
},
|
|
235
|
+
blockedReasons,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
export function assertLoraRunnerCompatibility(workload) {
|
|
240
|
+
const compatibility = getLoraRunnerCompatibility(workload);
|
|
241
|
+
if (compatibility.supported) return compatibility;
|
|
242
|
+
throw new Error([
|
|
243
|
+
'LoRA run is not supported by the current runner contract.',
|
|
244
|
+
`supported baseModelId="${compatibility.runnerContract.supportedBaseModelId}"`,
|
|
245
|
+
`supported datasetFormat="${compatibility.runnerContract.supportedDatasetFormat}"`,
|
|
246
|
+
`registered baseModelIds="${compatibility.runnerContract.registeredBaseModelIds.join(',')}"`,
|
|
247
|
+
`registered datasetFormats="${compatibility.runnerContract.registeredDatasetFormats.join(',')}"`,
|
|
248
|
+
`observed baseModelId="${compatibility.observed.baseModelId}"`,
|
|
249
|
+
`observed datasetFormat="${compatibility.observed.datasetFormat}"`,
|
|
250
|
+
`observed taskType="${compatibility.observed.taskType}"`,
|
|
251
|
+
`blockedReasons=${compatibility.blockedReasons.join(',')}`,
|
|
252
|
+
].join(' '));
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function summarizeTextPairLengths(rows) {
|
|
256
|
+
let minPromptChars = Number.POSITIVE_INFINITY;
|
|
257
|
+
let maxPromptChars = 0;
|
|
258
|
+
let minCompletionChars = Number.POSITIVE_INFINITY;
|
|
259
|
+
let maxCompletionChars = 0;
|
|
260
|
+
for (const row of rows) {
|
|
261
|
+
const promptChars = row.prompt.length;
|
|
262
|
+
const completionChars = row.completion.length;
|
|
263
|
+
minPromptChars = Math.min(minPromptChars, promptChars);
|
|
264
|
+
maxPromptChars = Math.max(maxPromptChars, promptChars);
|
|
265
|
+
minCompletionChars = Math.min(minCompletionChars, completionChars);
|
|
266
|
+
maxCompletionChars = Math.max(maxCompletionChars, completionChars);
|
|
267
|
+
}
|
|
268
|
+
if (!rows.length) {
|
|
269
|
+
minPromptChars = 0;
|
|
270
|
+
minCompletionChars = 0;
|
|
271
|
+
}
|
|
272
|
+
return {
|
|
273
|
+
minPromptChars,
|
|
274
|
+
maxPromptChars,
|
|
275
|
+
minCompletionChars,
|
|
276
|
+
maxCompletionChars,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
async function pathIsReadable(filePath) {
|
|
281
|
+
try {
|
|
282
|
+
await access(filePath);
|
|
283
|
+
return true;
|
|
284
|
+
} catch {
|
|
285
|
+
return false;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
async function resolveDatasetPathForLoadedWorkload(datasetPath, loadedWorkload) {
|
|
290
|
+
const source = String(datasetPath || '');
|
|
291
|
+
if (!source || isAbsolute(source) || /^https?:\/\//i.test(source)) {
|
|
292
|
+
return source;
|
|
293
|
+
}
|
|
294
|
+
const candidates = [];
|
|
295
|
+
const pushCandidate = (candidate) => {
|
|
296
|
+
if (candidate && !candidates.includes(candidate)) {
|
|
297
|
+
candidates.push(candidate);
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
pushCandidate(resolve(source));
|
|
301
|
+
let cursor = loadedWorkload?.absolutePath ? dirname(resolve(String(loadedWorkload.absolutePath))) : null;
|
|
302
|
+
while (cursor) {
|
|
303
|
+
pushCandidate(join(cursor, source));
|
|
304
|
+
const parent = dirname(cursor);
|
|
305
|
+
if (parent === cursor) break;
|
|
306
|
+
cursor = parent;
|
|
307
|
+
}
|
|
308
|
+
for (const candidate of candidates) {
|
|
309
|
+
if (await pathIsReadable(candidate)) {
|
|
310
|
+
return candidate;
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
return resolve(source);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
export async function preflightCausalLmLoraWorkload(workload, options = {}) {
|
|
317
|
+
const compatibility = getLoraRunnerCompatibility(workload);
|
|
318
|
+
if (!isCausalLmLoraWorkload(workload, compatibility)) {
|
|
319
|
+
throw new Error('preflightCausalLmLoraWorkload requires a causal-LM LoRA workload.');
|
|
320
|
+
}
|
|
321
|
+
if (!workload?.datasetPath) {
|
|
322
|
+
throw new Error('preflightCausalLmLoraWorkload requires workload.datasetPath.');
|
|
323
|
+
}
|
|
324
|
+
const datasetPath = options.datasetPath || workload.datasetPath;
|
|
325
|
+
const dataset = await loadTextPairsDataset(datasetPath, {
|
|
326
|
+
fetch: options.fetch,
|
|
327
|
+
readFile: options.readFile,
|
|
328
|
+
});
|
|
329
|
+
if (dataset.rowCount < 1) {
|
|
330
|
+
throw new Error(`Causal-LM LoRA dataset ${workload.datasetPath} has no rows.`);
|
|
331
|
+
}
|
|
332
|
+
const pipeline = getPipelineConfig(workload);
|
|
333
|
+
const adapter = pipeline.adapter || {};
|
|
334
|
+
return {
|
|
335
|
+
schemaVersion: 1,
|
|
336
|
+
supported: compatibility.supported,
|
|
337
|
+
runnerKey: compatibility.observed.runnerKey,
|
|
338
|
+
baseModelId: compatibility.observed.baseModelId,
|
|
339
|
+
baseModelFamily: compatibility.observed.baseModelFamily,
|
|
340
|
+
datasetPath: dataset.absolutePath,
|
|
341
|
+
datasetFormat: compatibility.observed.datasetFormat,
|
|
342
|
+
taskType: compatibility.observed.taskType,
|
|
343
|
+
rowCount: dataset.rowCount,
|
|
344
|
+
firstRowId: dataset.rows[0]?.id || null,
|
|
345
|
+
lastRowId: dataset.rows[dataset.rows.length - 1]?.id || null,
|
|
346
|
+
textPairFields: {
|
|
347
|
+
prompt: dataset.rows[0]?.promptField || null,
|
|
348
|
+
completion: dataset.rows[0]?.completionField || null,
|
|
349
|
+
},
|
|
350
|
+
textPairLengths: summarizeTextPairLengths(dataset.rows),
|
|
351
|
+
adapter: {
|
|
352
|
+
rank: Number(adapter.rank ?? 0),
|
|
353
|
+
alpha: Number(adapter.alpha ?? 0),
|
|
354
|
+
targetModules: normalizeLoraTargetModules(adapter),
|
|
355
|
+
},
|
|
356
|
+
blockedReasons: compatibility.blockedReasons.slice(),
|
|
357
|
+
};
|
|
358
|
+
}
|
|
31
359
|
|
|
32
360
|
function stableJson(value) {
|
|
33
361
|
return JSON.stringify(stableSortObject(value));
|
|
@@ -190,6 +518,165 @@ function createToyDatasetBatches(rows, batchSize) {
|
|
|
190
518
|
};
|
|
191
519
|
}
|
|
192
520
|
|
|
521
|
+
function createLossGradient(loss, lossScale) {
|
|
522
|
+
const lossElements = loss.shape.reduce((acc, value) => acc * value, 1);
|
|
523
|
+
const gradData = new Float32Array(lossElements);
|
|
524
|
+
gradData.fill(lossScale);
|
|
525
|
+
return createUploadedTensor(gradData, 'f32', loss.shape, 'lora_causal_lm_loss_grad');
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
function createCausalLmTrainingObjective() {
|
|
529
|
+
return {
|
|
530
|
+
name: 'causal_lm_cross_entropy',
|
|
531
|
+
async forward({ model, batch, tape }) {
|
|
532
|
+
if (typeof model.forwardCausalLm === 'function') {
|
|
533
|
+
return model.forwardCausalLm(batch, tape);
|
|
534
|
+
}
|
|
535
|
+
const logits = await model.forward(batch.input, tape);
|
|
536
|
+
return { logits };
|
|
537
|
+
},
|
|
538
|
+
async computeLoss({ batch, config, tape, forwardState }) {
|
|
539
|
+
const loss = await crossEntropyLoss(forwardState.logits, batch.targets, config, tape);
|
|
540
|
+
return { loss };
|
|
541
|
+
},
|
|
542
|
+
backwardTargets({ loss, lossScale }) {
|
|
543
|
+
return createLossGradient(loss, lossScale);
|
|
544
|
+
},
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
function createCausalLmDatasetBatches(samples) {
|
|
549
|
+
return {
|
|
550
|
+
async *batches() {
|
|
551
|
+
for (const sample of samples) {
|
|
552
|
+
const inputTensor = makeTensorFromUint32(
|
|
553
|
+
sample.inputIds,
|
|
554
|
+
[sample.inputIds.length],
|
|
555
|
+
`lora_causal_lm_input_${sample.id}`
|
|
556
|
+
);
|
|
557
|
+
const targetTensor = makeTensorFromUint32(
|
|
558
|
+
sample.targetIds,
|
|
559
|
+
[sample.targetIds.length],
|
|
560
|
+
`lora_causal_lm_target_${sample.id}`
|
|
561
|
+
);
|
|
562
|
+
try {
|
|
563
|
+
yield {
|
|
564
|
+
id: sample.id,
|
|
565
|
+
input: inputTensor,
|
|
566
|
+
targets: targetTensor,
|
|
567
|
+
prompt: sample.prompt,
|
|
568
|
+
completion: sample.completion,
|
|
569
|
+
};
|
|
570
|
+
} finally {
|
|
571
|
+
releaseTensor(inputTensor);
|
|
572
|
+
releaseTensor(targetTensor);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
},
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
async function loadCausalLmTextPairSamples(workload, datasetPath, tokenizer) {
|
|
580
|
+
const dataset = await loadTextPairsDataset(datasetPath);
|
|
581
|
+
const pipeline = getPipelineConfig(workload);
|
|
582
|
+
const maxLength = Math.floor(Number(
|
|
583
|
+
pipeline.maxLength
|
|
584
|
+
?? pipeline.sequenceLength
|
|
585
|
+
?? workload.training?.maxLength
|
|
586
|
+
));
|
|
587
|
+
if (!Number.isInteger(maxLength) || maxLength < 2) {
|
|
588
|
+
throw new Error('Causal-LM LoRA workload requires lora.maxLength or lora.sequenceLength >= 2.');
|
|
589
|
+
}
|
|
590
|
+
if (typeof pipeline.joinWith !== 'string') {
|
|
591
|
+
throw new Error('Causal-LM LoRA workload requires lora.joinWith.');
|
|
592
|
+
}
|
|
593
|
+
const joinWith = pipeline.joinWith;
|
|
594
|
+
const samples = await tokenizeTextPairs(tokenizer, dataset.rows, {
|
|
595
|
+
maxLength,
|
|
596
|
+
joinWith,
|
|
597
|
+
});
|
|
598
|
+
if (samples.length < 1) {
|
|
599
|
+
throw new Error(`Causal-LM LoRA dataset ${dataset.absolutePath} produced no tokenized samples.`);
|
|
600
|
+
}
|
|
601
|
+
return {
|
|
602
|
+
...dataset,
|
|
603
|
+
samples,
|
|
604
|
+
datasetHash: hashArtifactPayload({
|
|
605
|
+
rows: dataset.rows,
|
|
606
|
+
tokenization: {
|
|
607
|
+
maxLength,
|
|
608
|
+
joinWith,
|
|
609
|
+
},
|
|
610
|
+
}),
|
|
611
|
+
tokenization: {
|
|
612
|
+
maxLength,
|
|
613
|
+
joinWith,
|
|
614
|
+
sampleCount: samples.length,
|
|
615
|
+
},
|
|
616
|
+
};
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
async function createCausalLmLoraFixture(workload) {
|
|
620
|
+
const pipeline = getPipelineConfig(workload);
|
|
621
|
+
const adapter = pipeline.adapter || {};
|
|
622
|
+
const modelRef = getCausalLmBaseModelRef(workload);
|
|
623
|
+
const handle = await loadDistillModelHandle(modelRef, 'lora base', {
|
|
624
|
+
runtime: {
|
|
625
|
+
shared: {
|
|
626
|
+
debug: {
|
|
627
|
+
logLevel: {
|
|
628
|
+
defaultLogLevel: 'debug',
|
|
629
|
+
},
|
|
630
|
+
},
|
|
631
|
+
},
|
|
632
|
+
inference: {
|
|
633
|
+
compute: {
|
|
634
|
+
activationDtype: 'f32',
|
|
635
|
+
keepF32Weights: true,
|
|
636
|
+
},
|
|
637
|
+
},
|
|
638
|
+
},
|
|
639
|
+
});
|
|
640
|
+
let fixture = null;
|
|
641
|
+
try {
|
|
642
|
+
fixture = await createDistillStudentRuntimeModelFixture({
|
|
643
|
+
training: {
|
|
644
|
+
precision: workload.training?.precision || {},
|
|
645
|
+
},
|
|
646
|
+
}, {
|
|
647
|
+
distillRuntime: {
|
|
648
|
+
studentPipeline: handle.pipeline,
|
|
649
|
+
studentGraphMode: 'transformer_full',
|
|
650
|
+
},
|
|
651
|
+
studentGraphMode: 'transformer_full',
|
|
652
|
+
loraAdapter: {
|
|
653
|
+
rank: adapter.rank,
|
|
654
|
+
alpha: adapter.alpha,
|
|
655
|
+
targetModules: normalizeLoraTargetModules(adapter),
|
|
656
|
+
},
|
|
657
|
+
});
|
|
658
|
+
} catch (error) {
|
|
659
|
+
if (handle.pipeline && typeof handle.pipeline.unload === 'function') {
|
|
660
|
+
await handle.pipeline.unload();
|
|
661
|
+
}
|
|
662
|
+
throw error;
|
|
663
|
+
}
|
|
664
|
+
return {
|
|
665
|
+
model: fixture.model,
|
|
666
|
+
baseModelRef: modelRef,
|
|
667
|
+
baseModelUrl: handle.modelUrl || null,
|
|
668
|
+
baseManifest: handle.manifest || null,
|
|
669
|
+
tokenizer: handle.pipeline.tokenizer,
|
|
670
|
+
cleanup() {
|
|
671
|
+
fixture.cleanup();
|
|
672
|
+
if (handle.pipeline && typeof handle.pipeline.unload === 'function') {
|
|
673
|
+
return handle.pipeline.unload();
|
|
674
|
+
}
|
|
675
|
+
return undefined;
|
|
676
|
+
},
|
|
677
|
+
};
|
|
678
|
+
}
|
|
679
|
+
|
|
193
680
|
function collectProtectedBuffers(model) {
|
|
194
681
|
const protectedBuffers = new Set();
|
|
195
682
|
const groups = model.paramGroups();
|
|
@@ -328,29 +815,737 @@ function buildArtifact(loadedWorkload, options) {
|
|
|
328
815
|
configHash: options.configHash || workload.configHash,
|
|
329
816
|
});
|
|
330
817
|
return {
|
|
331
|
-
...payload,
|
|
332
|
-
artifactHash: hashArtifactPayload(payload),
|
|
818
|
+
...payload,
|
|
819
|
+
artifactHash: hashArtifactPayload(payload),
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
async function exportToyLoraModel(loadedWorkload, layout, model, checkpointId, checkpointStep, datasetHash) {
|
|
824
|
+
const workload = loadedWorkload.workload;
|
|
825
|
+
const targetModule = model.targetModule || workload.pipeline.adapter.targetModules[0];
|
|
826
|
+
const weightsFilename = `${checkpointId}.adapters.safetensors`;
|
|
827
|
+
const exported = await exportLoRAAdapter({
|
|
828
|
+
id: workload.pipeline.export?.id || `${workload.id}-${checkpointId}`,
|
|
829
|
+
name: workload.pipeline.export?.name || `${workload.id}-${checkpointId}`,
|
|
830
|
+
baseModel: workload.baseModelId,
|
|
831
|
+
rank: workload.pipeline.adapter.rank,
|
|
832
|
+
alpha: workload.pipeline.adapter.alpha,
|
|
833
|
+
targetModules: [targetModule],
|
|
834
|
+
tensors: [
|
|
835
|
+
{ name: `layers.0.${targetModule}.lora_a`, tensor: model.adapter.A },
|
|
836
|
+
{ name: `layers.0.${targetModule}.lora_b`, tensor: model.adapter.B },
|
|
837
|
+
],
|
|
838
|
+
weightsFormat: 'safetensors',
|
|
839
|
+
weightsPath: weightsFilename,
|
|
840
|
+
});
|
|
841
|
+
const manifestPath = join(layout.exports, `${checkpointId}.adapter.manifest.json`);
|
|
842
|
+
const weightsPath = join(layout.exports, weightsFilename);
|
|
843
|
+
if (!exported.weights) {
|
|
844
|
+
throw new Error('LoRA safetensors export did not return weights bytes.');
|
|
845
|
+
}
|
|
846
|
+
await writeFile(weightsPath, new Uint8Array(exported.weights));
|
|
847
|
+
await writeFile(manifestPath, exported.json, 'utf8');
|
|
848
|
+
await loadLoRAFromManifest(exported.manifest, {
|
|
849
|
+
readFile: async (filePath) => readFile(join(layout.exports, filePath)),
|
|
850
|
+
});
|
|
851
|
+
const artifactPayload = {
|
|
852
|
+
...buildArtifact(loadedWorkload, {
|
|
853
|
+
prefix: 'lora_export',
|
|
854
|
+
id: checkpointId,
|
|
855
|
+
artifactType: 'lora_adapter_manifest',
|
|
856
|
+
checkpointStep,
|
|
857
|
+
datasetHash,
|
|
858
|
+
}),
|
|
859
|
+
checkpointId,
|
|
860
|
+
manifestPath,
|
|
861
|
+
weightsPath,
|
|
862
|
+
weightsSha256: exported.weightsSha256 || null,
|
|
863
|
+
manifest: exported.manifest,
|
|
864
|
+
};
|
|
865
|
+
const artifactFile = await writeJsonArtifact(
|
|
866
|
+
join(layout.exports, `${checkpointId}.export.json`),
|
|
867
|
+
artifactPayload
|
|
868
|
+
);
|
|
869
|
+
return {
|
|
870
|
+
checkpointId,
|
|
871
|
+
manifestPath,
|
|
872
|
+
weightsPath,
|
|
873
|
+
weightsSha256: exported.weightsSha256 || null,
|
|
874
|
+
exportPath: artifactFile.path,
|
|
875
|
+
manifest: exported.manifest,
|
|
876
|
+
};
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
async function exportCausalLmLoraModel(loadedWorkload, layout, fixture, checkpointId, checkpointStep, datasetHash) {
|
|
880
|
+
const workload = loadedWorkload.workload;
|
|
881
|
+
const pipeline = getPipelineConfig(workload);
|
|
882
|
+
const adapter = pipeline.adapter || {};
|
|
883
|
+
const weightsFilename = `${checkpointId}.adapters.safetensors`;
|
|
884
|
+
const tensors = typeof fixture.model.loraTensorEntries === 'function'
|
|
885
|
+
? fixture.model.loraTensorEntries()
|
|
886
|
+
: [];
|
|
887
|
+
if (!tensors.length) {
|
|
888
|
+
throw new Error('Causal-LM LoRA export requires trained adapter tensors.');
|
|
889
|
+
}
|
|
890
|
+
const exported = await exportLoRAAdapter({
|
|
891
|
+
id: pipeline.export?.id || `${workload.id}-${checkpointId}`,
|
|
892
|
+
name: pipeline.export?.name || `${workload.id}-${checkpointId}`,
|
|
893
|
+
baseModel: workload.baseModelId,
|
|
894
|
+
rank: adapter.rank,
|
|
895
|
+
alpha: adapter.alpha,
|
|
896
|
+
targetModules: normalizeLoraTargetModules(adapter),
|
|
897
|
+
tensors,
|
|
898
|
+
weightsFormat: 'safetensors',
|
|
899
|
+
weightsPath: weightsFilename,
|
|
900
|
+
metadata: {
|
|
901
|
+
baseModelRef: fixture.baseModelRef,
|
|
902
|
+
baseModelUrl: fixture.baseModelUrl,
|
|
903
|
+
datasetFormat: pipeline.datasetFormat,
|
|
904
|
+
taskType: pipeline.taskType,
|
|
905
|
+
},
|
|
906
|
+
});
|
|
907
|
+
const manifestPath = join(layout.exports, `${checkpointId}.adapter.manifest.json`);
|
|
908
|
+
const runtimeManifestPath = join(layout.exports, 'runtime-adapter-manifest.json');
|
|
909
|
+
const weightsPath = join(layout.exports, weightsFilename);
|
|
910
|
+
if (!exported.weights) {
|
|
911
|
+
throw new Error('Causal-LM LoRA safetensors export did not return weights bytes.');
|
|
912
|
+
}
|
|
913
|
+
await writeFile(weightsPath, new Uint8Array(exported.weights));
|
|
914
|
+
await writeFile(manifestPath, exported.json, 'utf8');
|
|
915
|
+
await writeFile(runtimeManifestPath, exported.json, 'utf8');
|
|
916
|
+
await loadLoRAFromManifest(exported.manifest, {
|
|
917
|
+
readFile: async (filePath) => readFile(join(layout.exports, filePath)),
|
|
918
|
+
});
|
|
919
|
+
const artifactPayload = {
|
|
920
|
+
...buildArtifact(loadedWorkload, {
|
|
921
|
+
prefix: 'lora_export',
|
|
922
|
+
id: checkpointId,
|
|
923
|
+
artifactType: 'lora_adapter_manifest',
|
|
924
|
+
checkpointStep,
|
|
925
|
+
datasetHash,
|
|
926
|
+
}),
|
|
927
|
+
checkpointId,
|
|
928
|
+
manifestPath,
|
|
929
|
+
runtimeManifestPath,
|
|
930
|
+
weightsPath,
|
|
931
|
+
weightsSha256: exported.weightsSha256 || null,
|
|
932
|
+
manifest: exported.manifest,
|
|
933
|
+
};
|
|
934
|
+
const artifactFile = await writeJsonArtifact(
|
|
935
|
+
join(layout.exports, `${checkpointId}.export.json`),
|
|
936
|
+
artifactPayload
|
|
937
|
+
);
|
|
938
|
+
return {
|
|
939
|
+
checkpointId,
|
|
940
|
+
manifestPath,
|
|
941
|
+
runtimeManifestPath,
|
|
942
|
+
weightsPath,
|
|
943
|
+
weightsSha256: exported.weightsSha256 || null,
|
|
944
|
+
exportPath: artifactFile.path,
|
|
945
|
+
manifest: exported.manifest,
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
async function readLossMean(loss) {
|
|
950
|
+
const raw = await readBuffer(loss.buffer);
|
|
951
|
+
const data = loss.dtype === 'f16'
|
|
952
|
+
? f16ToF32Array(new Uint16Array(raw))
|
|
953
|
+
: new Float32Array(raw);
|
|
954
|
+
if (!data.length) return 0;
|
|
955
|
+
let sum = 0;
|
|
956
|
+
for (const value of data) {
|
|
957
|
+
sum += Number.isFinite(value) ? value : 0;
|
|
958
|
+
}
|
|
959
|
+
return sum / data.length;
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
function finiteMetric(value) {
|
|
963
|
+
const parsed = Number(value);
|
|
964
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
965
|
+
}
|
|
966
|
+
|
|
967
|
+
function buildLossQualityClaim(evalDataset, meanLoss, baselineReport) {
|
|
968
|
+
const baselineLoss = finiteMetric(baselineReport?.loss ?? baselineReport?.primaryScore);
|
|
969
|
+
const adapterLoss = finiteMetric(meanLoss);
|
|
970
|
+
if (baselineLoss === null || adapterLoss === null) {
|
|
971
|
+
return null;
|
|
972
|
+
}
|
|
973
|
+
const quality = evalDataset.quality || null;
|
|
974
|
+
const absoluteImprovement = baselineLoss - adapterLoss;
|
|
975
|
+
const relativeImprovement = baselineLoss === 0 ? 0 : absoluteImprovement / Math.abs(baselineLoss);
|
|
976
|
+
const minAbsoluteImprovement = finiteMetric(quality?.minAbsoluteImprovement) ?? 0;
|
|
977
|
+
const minRelativeImprovement = finiteMetric(quality?.minRelativeImprovement) ?? 0;
|
|
978
|
+
const improved = absoluteImprovement >= minAbsoluteImprovement
|
|
979
|
+
&& relativeImprovement >= minRelativeImprovement;
|
|
980
|
+
return {
|
|
981
|
+
baseline: quality?.baseline || 'base_model',
|
|
982
|
+
metric: 'loss',
|
|
983
|
+
selectionGoal: 'min',
|
|
984
|
+
adapterScore: adapterLoss,
|
|
985
|
+
baselineScore: baselineLoss,
|
|
986
|
+
delta: adapterLoss - baselineLoss,
|
|
987
|
+
absoluteImprovement,
|
|
988
|
+
relativeImprovement,
|
|
989
|
+
minAbsoluteImprovement,
|
|
990
|
+
minRelativeImprovement,
|
|
991
|
+
improved,
|
|
992
|
+
requireImprovement: quality?.requireImprovement === true,
|
|
993
|
+
};
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
async function computeCausalLmTextPairLosses(workload, fixture, samples, protectedBuffers) {
|
|
997
|
+
const losses = [];
|
|
998
|
+
for await (const resolvedBatch of createCausalLmDatasetBatches(samples).batches()) {
|
|
999
|
+
const tape = new AutogradTape(loadBackwardRegistry());
|
|
1000
|
+
let loss = null;
|
|
1001
|
+
try {
|
|
1002
|
+
const { logits } = await fixture.model.forwardCausalLm(resolvedBatch, tape);
|
|
1003
|
+
loss = await crossEntropyLoss(logits, resolvedBatch.targets, { training: { precision: workload.training.precision } }, tape);
|
|
1004
|
+
losses.push(await readLossMean(loss));
|
|
1005
|
+
} finally {
|
|
1006
|
+
disposeTapeOutputs(tape, protectedBuffers);
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
return losses;
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
async function evaluateCausalLmLoraModel(workload, fixture, dataset, layout = null, checkpointMeta = {}) {
|
|
1013
|
+
const protectedBuffers = collectProtectedBuffers(fixture.model);
|
|
1014
|
+
const evalReports = [];
|
|
1015
|
+
const evalDatasets = Array.isArray(workload.evalDatasets) ? workload.evalDatasets : [];
|
|
1016
|
+
for (const evalDataset of evalDatasets) {
|
|
1017
|
+
if (evalDataset.evalKind !== 'text_generation') {
|
|
1018
|
+
throw new Error(`Causal-LM LoRA eval supports text_generation only, got "${evalDataset.evalKind}".`);
|
|
1019
|
+
}
|
|
1020
|
+
const evalDatasetPath = await resolveDatasetPathForLoadedWorkload(evalDataset.datasetPath, {
|
|
1021
|
+
absolutePath: checkpointMeta.workloadPath || null,
|
|
1022
|
+
});
|
|
1023
|
+
const evalDatasetMaterialized = evalDataset.datasetPath === dataset.absolutePath
|
|
1024
|
+
? dataset
|
|
1025
|
+
: await loadCausalLmTextPairSamples(workload, evalDatasetPath, fixture.tokenizer);
|
|
1026
|
+
const losses = await computeCausalLmTextPairLosses(
|
|
1027
|
+
workload,
|
|
1028
|
+
fixture,
|
|
1029
|
+
evalDatasetMaterialized.samples,
|
|
1030
|
+
protectedBuffers
|
|
1031
|
+
);
|
|
1032
|
+
const meanLoss = losses.length
|
|
1033
|
+
? losses.reduce((sum, value) => sum + value, 0) / losses.length
|
|
1034
|
+
: null;
|
|
1035
|
+
const baselineReports = checkpointMeta.baselineReportsByEvalDatasetId || {};
|
|
1036
|
+
const baselineReport = baselineReports[evalDataset.id] || null;
|
|
1037
|
+
const qualityClaim = buildLossQualityClaim(evalDataset, meanLoss, baselineReport);
|
|
1038
|
+
const reportPayload = {
|
|
1039
|
+
artifactType: 'training_eval_report',
|
|
1040
|
+
schemaVersion: 1,
|
|
1041
|
+
generatedAt: new Date().toISOString(),
|
|
1042
|
+
workloadId: workload.id,
|
|
1043
|
+
workloadPath: checkpointMeta.workloadPath || null,
|
|
1044
|
+
workloadSha256: checkpointMeta.workloadSha256 || null,
|
|
1045
|
+
configHash: checkpointMeta.configHash || workload.configHash,
|
|
1046
|
+
datasetPath: evalDataset.datasetPath,
|
|
1047
|
+
datasetHash: evalDatasetMaterialized.datasetHash,
|
|
1048
|
+
baseModelId: workload.baseModelId,
|
|
1049
|
+
baseModelRef: fixture.baseModelRef,
|
|
1050
|
+
stage: checkpointMeta.stage || 'lora',
|
|
1051
|
+
checkpointStep: checkpointMeta.checkpointStep ?? null,
|
|
1052
|
+
checkpointId: checkpointMeta.checkpointId || null,
|
|
1053
|
+
evalDatasetId: evalDataset.id,
|
|
1054
|
+
metrics: {
|
|
1055
|
+
loss: {
|
|
1056
|
+
score: meanLoss,
|
|
1057
|
+
samples: losses.length,
|
|
1058
|
+
},
|
|
1059
|
+
},
|
|
1060
|
+
primaryMetric: 'loss',
|
|
1061
|
+
primaryScore: meanLoss,
|
|
1062
|
+
loss: meanLoss,
|
|
1063
|
+
baseline: baselineReport
|
|
1064
|
+
? {
|
|
1065
|
+
checkpointId: baselineReport.checkpointId || null,
|
|
1066
|
+
stage: baselineReport.stage || null,
|
|
1067
|
+
primaryMetric: baselineReport.primaryMetric || null,
|
|
1068
|
+
primaryScore: baselineReport.primaryScore ?? null,
|
|
1069
|
+
loss: baselineReport.loss ?? null,
|
|
1070
|
+
reportPath: baselineReport.reportPath || null,
|
|
1071
|
+
}
|
|
1072
|
+
: null,
|
|
1073
|
+
qualityClaim,
|
|
1074
|
+
};
|
|
1075
|
+
const reportFile = layout
|
|
1076
|
+
? await writeJsonArtifact(
|
|
1077
|
+
join(layout.eval, `${checkpointMeta.checkpointId || 'checkpoint'}__${evalDataset.id}.json`),
|
|
1078
|
+
reportPayload
|
|
1079
|
+
)
|
|
1080
|
+
: null;
|
|
1081
|
+
evalReports.push({
|
|
1082
|
+
...reportPayload,
|
|
1083
|
+
reportPath: reportFile?.path || null,
|
|
1084
|
+
});
|
|
1085
|
+
}
|
|
1086
|
+
return evalReports;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
function getCausalLmFreezeConfig(workload) {
|
|
1090
|
+
const pipeline = getPipelineConfig(workload);
|
|
1091
|
+
const freeze = pipeline.freeze;
|
|
1092
|
+
if (!freeze || typeof freeze !== 'object' || Array.isArray(freeze)) {
|
|
1093
|
+
throw new Error('Causal-LM LoRA workload requires lora.freeze.');
|
|
1094
|
+
}
|
|
1095
|
+
return {
|
|
1096
|
+
encoder: freeze.encoder === true,
|
|
1097
|
+
prior: freeze.prior === true,
|
|
1098
|
+
decoder: freeze.decoder === true,
|
|
1099
|
+
base: freeze.base === true,
|
|
1100
|
+
lora: freeze.lora === true,
|
|
1101
|
+
};
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
function createLoraRunnerTrainingConfig(workload, freeze) {
|
|
1105
|
+
return {
|
|
1106
|
+
training: {
|
|
1107
|
+
enabled: true,
|
|
1108
|
+
optimizer: {
|
|
1109
|
+
type: workload.training.optimizer.type,
|
|
1110
|
+
lr: workload.training.optimizer.lr,
|
|
1111
|
+
beta1: workload.training.optimizer.beta1,
|
|
1112
|
+
beta2: workload.training.optimizer.beta2,
|
|
1113
|
+
eps: workload.training.optimizer.eps,
|
|
1114
|
+
weightDecay: workload.training.optimizer.weightDecay,
|
|
1115
|
+
scheduler: workload.training.optimizer.scheduler,
|
|
1116
|
+
},
|
|
1117
|
+
gradient: {
|
|
1118
|
+
maxNorm: workload.training.gradientClipping.maxNorm,
|
|
1119
|
+
},
|
|
1120
|
+
precision: workload.training.precision,
|
|
1121
|
+
lossScaling: { enabled: false },
|
|
1122
|
+
distill: {
|
|
1123
|
+
enabled: false,
|
|
1124
|
+
stage: 'stage_a',
|
|
1125
|
+
teacherModelId: null,
|
|
1126
|
+
studentModelId: null,
|
|
1127
|
+
datasetId: null,
|
|
1128
|
+
datasetPath: null,
|
|
1129
|
+
languagePair: null,
|
|
1130
|
+
sourceLangs: null,
|
|
1131
|
+
targetLangs: null,
|
|
1132
|
+
pairAllowlist: null,
|
|
1133
|
+
strictPairContract: false,
|
|
1134
|
+
shardIndex: null,
|
|
1135
|
+
shardCount: null,
|
|
1136
|
+
resumeFrom: null,
|
|
1137
|
+
artifactDir: null,
|
|
1138
|
+
stageAArtifact: null,
|
|
1139
|
+
stageAArtifactHash: null,
|
|
1140
|
+
temperature: 1,
|
|
1141
|
+
alphaKd: 1,
|
|
1142
|
+
alphaCe: 0,
|
|
1143
|
+
allowHintFallback: false,
|
|
1144
|
+
tripletMargin: 0.2,
|
|
1145
|
+
studentGraphMode: 'transformer_full',
|
|
1146
|
+
freeze,
|
|
1147
|
+
},
|
|
1148
|
+
ul: {
|
|
1149
|
+
enabled: false,
|
|
1150
|
+
stage: 'stage1_joint',
|
|
1151
|
+
stage1Artifact: null,
|
|
1152
|
+
stage1ArtifactHash: null,
|
|
1153
|
+
artifactDir: null,
|
|
1154
|
+
lambda0: 5,
|
|
1155
|
+
seed: workload.seed,
|
|
1156
|
+
noiseSchedule: { name: 'linear', minSigma: 0.1, maxSigma: 1, steps: 1 },
|
|
1157
|
+
priorAlignment: { enabled: false, weight: 1 },
|
|
1158
|
+
decoderSigmoidWeight: { enabled: false, maxWeight: 1 },
|
|
1159
|
+
lossWeights: { prior: 1, decoder: 1, recon: 1 },
|
|
1160
|
+
freeze: null,
|
|
1161
|
+
},
|
|
1162
|
+
},
|
|
1163
|
+
};
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
function createLoraOptimizer(workload) {
|
|
1167
|
+
return new AdamOptimizer({
|
|
1168
|
+
training: {
|
|
1169
|
+
optimizer: {
|
|
1170
|
+
type: workload.training.optimizer.type,
|
|
1171
|
+
lr: workload.training.optimizer.lr,
|
|
1172
|
+
beta1: workload.training.optimizer.beta1,
|
|
1173
|
+
beta2: workload.training.optimizer.beta2,
|
|
1174
|
+
eps: workload.training.optimizer.eps,
|
|
1175
|
+
weightDecay: workload.training.optimizer.weightDecay,
|
|
1176
|
+
scheduler: workload.training.optimizer.scheduler,
|
|
1177
|
+
},
|
|
1178
|
+
gradient: {
|
|
1179
|
+
maxNorm: workload.training.gradientClipping.maxNorm,
|
|
1180
|
+
},
|
|
1181
|
+
precision: workload.training.precision,
|
|
1182
|
+
},
|
|
1183
|
+
});
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
async function createLoraRunLayout(options, workload) {
|
|
1187
|
+
const layout = options.runRoot
|
|
1188
|
+
? {
|
|
1189
|
+
runRoot: resolve(String(options.runRoot)),
|
|
1190
|
+
logs: join(resolve(String(options.runRoot)), 'logs'),
|
|
1191
|
+
checkpoints: join(resolve(String(options.runRoot)), 'checkpoints'),
|
|
1192
|
+
eval: join(resolve(String(options.runRoot)), 'eval'),
|
|
1193
|
+
scoreboard: join(resolve(String(options.runRoot)), 'scoreboard'),
|
|
1194
|
+
exports: join(resolve(String(options.runRoot)), 'exports'),
|
|
1195
|
+
compare: join(resolve(String(options.runRoot)), 'compare'),
|
|
1196
|
+
qualityGate: join(resolve(String(options.runRoot)), 'quality-gate'),
|
|
1197
|
+
}
|
|
1198
|
+
: await createTrainingRunLayout({
|
|
1199
|
+
kind: 'lora',
|
|
1200
|
+
workloadId: workload.id,
|
|
1201
|
+
timestamp: options.timestamp || null,
|
|
1202
|
+
});
|
|
1203
|
+
await Promise.all(Object.values(layout).map((dirPath) => mkdir(dirPath, { recursive: true })));
|
|
1204
|
+
return layout;
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
function isObjectRecord(value) {
|
|
1208
|
+
return value && typeof value === 'object' && !Array.isArray(value);
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
function hashTextPairsDataset(dataset) {
|
|
1212
|
+
return hashArtifactPayload({ rows: dataset.rows });
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
function parseCausalLmLoraTensorName(name) {
|
|
1216
|
+
const text = String(name || '');
|
|
1217
|
+
const match = text.match(/(?:^|\.)layers?\.?(\d+)\.(?:[A-Za-z0-9_]+\.)*([A-Za-z0-9_]+)\.lora_([ab])(?:\.[A-Za-z0-9_]+)?$/i);
|
|
1218
|
+
if (!match) return null;
|
|
1219
|
+
const layer = Number.parseInt(match[1], 10);
|
|
1220
|
+
const rawModule = match[2].toLowerCase();
|
|
1221
|
+
const module = LORA_MODULE_ALIASES[rawModule];
|
|
1222
|
+
if (!module) return null;
|
|
1223
|
+
return {
|
|
1224
|
+
layer,
|
|
1225
|
+
module,
|
|
1226
|
+
kind: match[3].toLowerCase() === 'a' ? 'a' : 'b',
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
function normalizeTensorShape(value, label) {
|
|
1231
|
+
const shape = Array.isArray(value) ? value.map((entry) => Number(entry)) : [];
|
|
1232
|
+
if (shape.length !== 2 || shape.some((entry) => !Number.isInteger(entry) || entry < 1)) {
|
|
1233
|
+
throw new Error(`${label} requires shape [rows, cols].`);
|
|
1234
|
+
}
|
|
1235
|
+
return shape;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
function normalizeTrainerTensor(entry, index) {
|
|
1239
|
+
if (!isObjectRecord(entry)) {
|
|
1240
|
+
throw new Error(`Causal-LM trainer tensor ${index + 1} must be an object.`);
|
|
1241
|
+
}
|
|
1242
|
+
const name = String(entry.name || '').trim();
|
|
1243
|
+
if (!name) {
|
|
1244
|
+
throw new Error(`Causal-LM trainer tensor ${index + 1} requires name.`);
|
|
1245
|
+
}
|
|
1246
|
+
const tensor = entry.tensor ?? entry.data ?? entry.values;
|
|
1247
|
+
if (tensor === undefined || tensor === null) {
|
|
1248
|
+
throw new Error(`Causal-LM trainer tensor "${name}" requires tensor data.`);
|
|
1249
|
+
}
|
|
1250
|
+
const shape = normalizeTensorShape(entry.shape ?? tensor?.shape, `Causal-LM trainer tensor "${name}"`);
|
|
1251
|
+
let normalizedTensor = tensor;
|
|
1252
|
+
if (Array.isArray(tensor)) {
|
|
1253
|
+
normalizedTensor = new Float32Array(tensor.map((value) => Number(value)));
|
|
1254
|
+
}
|
|
1255
|
+
if (normalizedTensor instanceof Float32Array && normalizedTensor.length !== shape[0] * shape[1]) {
|
|
1256
|
+
throw new Error(
|
|
1257
|
+
`Causal-LM trainer tensor "${name}" shape mismatch: expected ${shape[0] * shape[1]}, got ${normalizedTensor.length}.`
|
|
1258
|
+
);
|
|
1259
|
+
}
|
|
1260
|
+
return {
|
|
1261
|
+
name,
|
|
1262
|
+
shape,
|
|
1263
|
+
dtype: entry.dtype || 'f32',
|
|
1264
|
+
tensor: normalizedTensor,
|
|
1265
|
+
};
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
function assertCausalLmTensorCoverage(tensors, adapter) {
|
|
1269
|
+
const targetModules = normalizeLoraTargetModules(adapter);
|
|
1270
|
+
if (!targetModules.length) {
|
|
1271
|
+
throw new Error('Causal-LM LoRA export requires at least one target module.');
|
|
1272
|
+
}
|
|
1273
|
+
const targetSet = new Set(targetModules);
|
|
1274
|
+
const layerModules = new Map();
|
|
1275
|
+
const seenModules = new Set();
|
|
1276
|
+
for (const tensor of tensors) {
|
|
1277
|
+
const parsed = parseCausalLmLoraTensorName(tensor.name);
|
|
1278
|
+
if (!parsed) {
|
|
1279
|
+
throw new Error(`Unrecognized Causal-LM LoRA tensor name: ${tensor.name}`);
|
|
1280
|
+
}
|
|
1281
|
+
if (!targetSet.has(parsed.module)) {
|
|
1282
|
+
throw new Error(
|
|
1283
|
+
`Causal-LM LoRA tensor "${tensor.name}" targets module "${parsed.module}" outside workload targetModules.`
|
|
1284
|
+
);
|
|
1285
|
+
}
|
|
1286
|
+
seenModules.add(parsed.module);
|
|
1287
|
+
if (!layerModules.has(parsed.layer)) {
|
|
1288
|
+
layerModules.set(parsed.layer, new Map());
|
|
1289
|
+
}
|
|
1290
|
+
const modules = layerModules.get(parsed.layer);
|
|
1291
|
+
if (!modules.has(parsed.module)) {
|
|
1292
|
+
modules.set(parsed.module, new Set());
|
|
1293
|
+
}
|
|
1294
|
+
modules.get(parsed.module).add(parsed.kind);
|
|
1295
|
+
}
|
|
1296
|
+
if (layerModules.size === 0) {
|
|
1297
|
+
throw new Error('Causal-LM trainer returned no LoRA layer tensors.');
|
|
1298
|
+
}
|
|
1299
|
+
for (const moduleName of targetModules) {
|
|
1300
|
+
if (!seenModules.has(moduleName)) {
|
|
1301
|
+
throw new Error(`Causal-LM trainer returned no tensors for target module "${moduleName}".`);
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
for (const [layerIndex, modules] of layerModules.entries()) {
|
|
1305
|
+
for (const moduleName of targetModules) {
|
|
1306
|
+
const kinds = modules.get(moduleName);
|
|
1307
|
+
if (!kinds?.has('a') || !kinds?.has('b')) {
|
|
1308
|
+
throw new Error(
|
|
1309
|
+
`Causal-LM trainer layer ${layerIndex} module ${moduleName} must include both lora_a and lora_b tensors.`
|
|
1310
|
+
);
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
function normalizeCausalLmTrainerOutput(output, workload) {
|
|
1317
|
+
if (!isObjectRecord(output)) {
|
|
1318
|
+
throw new Error('Causal-LM trainer must return an object.');
|
|
1319
|
+
}
|
|
1320
|
+
const rawTensors = Array.isArray(output.tensors)
|
|
1321
|
+
? output.tensors
|
|
1322
|
+
: (Array.isArray(output.weights) ? output.weights : null);
|
|
1323
|
+
if (!rawTensors || rawTensors.length === 0) {
|
|
1324
|
+
throw new Error('Causal-LM trainer must return non-empty tensors.');
|
|
1325
|
+
}
|
|
1326
|
+
const tensors = rawTensors.map((entry, index) => normalizeTrainerTensor(entry, index));
|
|
1327
|
+
assertCausalLmTensorCoverage(tensors, getPipelineConfig(workload).adapter);
|
|
1328
|
+
const checkpointStep = Number.isInteger(Number(output.checkpointStep))
|
|
1329
|
+
? Number(output.checkpointStep)
|
|
1330
|
+
: Number(workload.training?.steps || 0);
|
|
1331
|
+
const checkpointId = String(output.checkpointId || `checkpoint-${String(checkpointStep).padStart(6, '0')}`).trim();
|
|
1332
|
+
if (!/^[A-Za-z0-9_-]+$/.test(checkpointId)) {
|
|
1333
|
+
throw new Error(`Causal-LM trainer checkpointId "${checkpointId}" must use alphanumeric, underscore, or hyphen characters.`);
|
|
1334
|
+
}
|
|
1335
|
+
return {
|
|
1336
|
+
checkpointId,
|
|
1337
|
+
checkpointStep,
|
|
1338
|
+
adapterId: String(output.adapterId || '').trim() || null,
|
|
1339
|
+
adapterName: String(output.adapterName || '').trim() || null,
|
|
1340
|
+
trainerId: String(output.trainerId || '').trim() || null,
|
|
1341
|
+
runnerId: String(output.runnerId || '').trim() || null,
|
|
1342
|
+
metrics: isObjectRecord(output.metrics) ? output.metrics : {},
|
|
1343
|
+
receipts: Array.isArray(output.receipts) ? output.receipts.slice() : [],
|
|
1344
|
+
evalReports: Array.isArray(output.evalReports) ? output.evalReports.slice() : [],
|
|
1345
|
+
tensors,
|
|
1346
|
+
};
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
function normalizeProviderEvalReport(entry, index, context) {
|
|
1350
|
+
if (!isObjectRecord(entry)) {
|
|
1351
|
+
throw new Error(`Causal-LM trainer evalReports[${index}] must be an object.`);
|
|
1352
|
+
}
|
|
1353
|
+
const evalDatasetId = String(entry.evalDatasetId || entry.id || '').trim();
|
|
1354
|
+
if (!evalDatasetId) {
|
|
1355
|
+
throw new Error(`Causal-LM trainer evalReports[${index}].evalDatasetId is required.`);
|
|
1356
|
+
}
|
|
1357
|
+
const primaryMetric = String(entry.primaryMetric || 'loss').trim();
|
|
1358
|
+
if (!primaryMetric) {
|
|
1359
|
+
throw new Error(`Causal-LM trainer evalReports[${index}].primaryMetric is required.`);
|
|
1360
|
+
}
|
|
1361
|
+
const primaryScore = finiteMetric(entry.primaryScore ?? entry.score ?? entry.loss);
|
|
1362
|
+
if (primaryScore === null) {
|
|
1363
|
+
throw new Error(`Causal-LM trainer evalReports[${index}].primaryScore must be finite.`);
|
|
1364
|
+
}
|
|
1365
|
+
const loss = finiteMetric(entry.loss ?? (primaryMetric === 'loss' ? primaryScore : null));
|
|
1366
|
+
const baselineScore = finiteMetric(entry.baseline?.primaryScore ?? entry.baseline?.loss);
|
|
1367
|
+
const qualityClaim = isObjectRecord(entry.qualityClaim)
|
|
1368
|
+
? entry.qualityClaim
|
|
1369
|
+
: (baselineScore === null || loss === null
|
|
1370
|
+
? null
|
|
1371
|
+
: {
|
|
1372
|
+
baseline: entry.baseline?.stage || 'base_model',
|
|
1373
|
+
metric: 'loss',
|
|
1374
|
+
selectionGoal: 'min',
|
|
1375
|
+
adapterScore: loss,
|
|
1376
|
+
baselineScore,
|
|
1377
|
+
delta: loss - baselineScore,
|
|
1378
|
+
absoluteImprovement: baselineScore - loss,
|
|
1379
|
+
relativeImprovement: baselineScore === 0 ? 0 : (baselineScore - loss) / Math.abs(baselineScore),
|
|
1380
|
+
minAbsoluteImprovement: 0,
|
|
1381
|
+
minRelativeImprovement: 0,
|
|
1382
|
+
improved: loss <= baselineScore,
|
|
1383
|
+
requireImprovement: false,
|
|
1384
|
+
});
|
|
1385
|
+
const agentEval = isObjectRecord(entry.agentEval)
|
|
1386
|
+
? entry.agentEval
|
|
1387
|
+
: (isObjectRecord(entry.heldoutGate) ? entry.heldoutGate : null);
|
|
1388
|
+
return {
|
|
1389
|
+
artifactType: 'training_eval_report',
|
|
1390
|
+
schemaVersion: 1,
|
|
1391
|
+
generatedAt: new Date().toISOString(),
|
|
1392
|
+
workloadId: context.workload.id,
|
|
1393
|
+
workloadPath: context.loadedWorkload.absolutePath || null,
|
|
1394
|
+
workloadSha256: context.loadedWorkload.workloadSha256 || null,
|
|
1395
|
+
configHash: context.workload.configHash,
|
|
1396
|
+
datasetPath: entry.datasetPath || context.dataset.absolutePath,
|
|
1397
|
+
datasetHash: entry.datasetHash || context.datasetHash,
|
|
1398
|
+
baseModelId: context.workload.baseModelId,
|
|
1399
|
+
baseModelRef: context.preflight.baseModelId,
|
|
1400
|
+
stage: entry.stage || 'lora',
|
|
1401
|
+
checkpointId: context.checkpointId,
|
|
1402
|
+
checkpointStep: context.checkpointStep,
|
|
1403
|
+
evalDatasetId,
|
|
1404
|
+
metrics: isObjectRecord(entry.metrics)
|
|
1405
|
+
? entry.metrics
|
|
1406
|
+
: {
|
|
1407
|
+
[primaryMetric]: {
|
|
1408
|
+
score: primaryScore,
|
|
1409
|
+
samples: finiteMetric(entry.samples) ?? context.dataset.rowCount,
|
|
1410
|
+
},
|
|
1411
|
+
},
|
|
1412
|
+
primaryMetric,
|
|
1413
|
+
primaryScore,
|
|
1414
|
+
loss,
|
|
1415
|
+
baseline: isObjectRecord(entry.baseline) ? entry.baseline : null,
|
|
1416
|
+
qualityClaim,
|
|
1417
|
+
agentEval,
|
|
1418
|
+
};
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
async function writeProviderEvalReports(context) {
|
|
1422
|
+
const reports = [];
|
|
1423
|
+
for (let index = 0; index < context.trainerOutput.evalReports.length; index += 1) {
|
|
1424
|
+
const report = normalizeProviderEvalReport(context.trainerOutput.evalReports[index], index, context);
|
|
1425
|
+
const reportFile = await writeJsonArtifact(
|
|
1426
|
+
join(context.layout.eval, `${context.checkpointId}__${report.evalDatasetId}.json`),
|
|
1427
|
+
report
|
|
1428
|
+
);
|
|
1429
|
+
const materialized = {
|
|
1430
|
+
...report,
|
|
1431
|
+
reportPath: reportFile.path,
|
|
1432
|
+
};
|
|
1433
|
+
reports.push(materialized);
|
|
1434
|
+
await appendScoreboardRow(context.layout.scoreboard, {
|
|
1435
|
+
artifactType: 'training_scoreboard',
|
|
1436
|
+
schemaVersion: 1,
|
|
1437
|
+
generatedAt: new Date().toISOString(),
|
|
1438
|
+
checkpointId: materialized.checkpointId,
|
|
1439
|
+
checkpointStep: materialized.checkpointStep,
|
|
1440
|
+
stage: materialized.stage,
|
|
1441
|
+
evalDatasetId: materialized.evalDatasetId,
|
|
1442
|
+
primaryMetric: materialized.primaryMetric,
|
|
1443
|
+
primaryScore: materialized.primaryScore,
|
|
1444
|
+
loss: materialized.loss,
|
|
1445
|
+
qualityClaim: materialized.qualityClaim || null,
|
|
1446
|
+
agentEval: materialized.agentEval || null,
|
|
1447
|
+
metrics: {
|
|
1448
|
+
[materialized.primaryMetric]: materialized.primaryScore,
|
|
1449
|
+
primaryScore: materialized.primaryScore,
|
|
1450
|
+
agent_heldout_gate: materialized.agentEval?.passRate ?? null,
|
|
1451
|
+
},
|
|
1452
|
+
}, {
|
|
1453
|
+
selectionMetric: context.workload.selectionMetric,
|
|
1454
|
+
selectionGoal: context.workload.selectionGoal,
|
|
1455
|
+
});
|
|
1456
|
+
}
|
|
1457
|
+
return reports;
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
async function resolveCausalLmTrainer(loadedWorkload, options = {}) {
|
|
1461
|
+
if (typeof options.causalLmTrainer === 'function') {
|
|
1462
|
+
return {
|
|
1463
|
+
train: options.causalLmTrainer,
|
|
1464
|
+
runnerId: options.causalLmTrainer.runnerId || 'injected_causal_lm_lora_trainer',
|
|
1465
|
+
source: 'runLoraPipeline.options.causalLmTrainer',
|
|
1466
|
+
exportName: null,
|
|
1467
|
+
};
|
|
1468
|
+
}
|
|
1469
|
+
const trainerConfig = getPipelineConfig(loadedWorkload.workload).trainer;
|
|
1470
|
+
if (!trainerConfig) {
|
|
1471
|
+
throw new Error(
|
|
1472
|
+
'causal_lm_trainer_not_configured: provide runLoraPipeline({ causalLmTrainer }) or lora.trainer.modulePath in the workload.'
|
|
1473
|
+
);
|
|
1474
|
+
}
|
|
1475
|
+
const modulePath = String(trainerConfig.modulePath || trainerConfig.path || '').trim();
|
|
1476
|
+
if (!modulePath) {
|
|
1477
|
+
throw new Error('causal_lm_trainer_not_configured: lora.trainer.modulePath is required.');
|
|
1478
|
+
}
|
|
1479
|
+
const exportName = String(trainerConfig.exportName || 'trainCausalLmLora').trim();
|
|
1480
|
+
if (!exportName) {
|
|
1481
|
+
throw new Error('causal_lm_trainer_not_configured: lora.trainer.exportName is required.');
|
|
1482
|
+
}
|
|
1483
|
+
const workloadDir = loadedWorkload.absolutePath
|
|
1484
|
+
? dirname(resolve(String(loadedWorkload.absolutePath)))
|
|
1485
|
+
: process.cwd();
|
|
1486
|
+
const absoluteModulePath = isAbsolute(modulePath)
|
|
1487
|
+
? modulePath
|
|
1488
|
+
: resolve(workloadDir, modulePath);
|
|
1489
|
+
const trainerModule = await import(pathToFileURL(absoluteModulePath).href);
|
|
1490
|
+
const train = trainerModule[exportName];
|
|
1491
|
+
if (typeof train !== 'function') {
|
|
1492
|
+
throw new Error(`causal_lm_trainer_not_configured: ${absoluteModulePath} does not export ${exportName}().`);
|
|
1493
|
+
}
|
|
1494
|
+
return {
|
|
1495
|
+
train,
|
|
1496
|
+
runnerId: String(trainerConfig.runnerId || exportName).trim(),
|
|
1497
|
+
source: absoluteModulePath,
|
|
1498
|
+
exportName,
|
|
333
1499
|
};
|
|
334
1500
|
}
|
|
335
1501
|
|
|
336
|
-
async function
|
|
1502
|
+
async function exportProviderCausalLmLoraModel(
|
|
1503
|
+
loadedWorkload,
|
|
1504
|
+
layout,
|
|
1505
|
+
trainerOutput,
|
|
1506
|
+
checkpointId,
|
|
1507
|
+
checkpointStep,
|
|
1508
|
+
datasetHash,
|
|
1509
|
+
trainerInfo,
|
|
1510
|
+
preflight
|
|
1511
|
+
) {
|
|
337
1512
|
const workload = loadedWorkload.workload;
|
|
338
|
-
const
|
|
1513
|
+
const pipeline = getPipelineConfig(workload);
|
|
1514
|
+
const weightsFilename = `${checkpointId}.adapters.safetensors`;
|
|
339
1515
|
const exported = await exportLoRAAdapter({
|
|
340
|
-
id:
|
|
341
|
-
name:
|
|
1516
|
+
id: pipeline.export?.id || trainerOutput.adapterId || `${workload.id}-${checkpointId}`,
|
|
1517
|
+
name: pipeline.export?.name || trainerOutput.adapterName || `${workload.id}-${checkpointId}`,
|
|
342
1518
|
baseModel: workload.baseModelId,
|
|
343
|
-
rank:
|
|
344
|
-
alpha:
|
|
345
|
-
targetModules:
|
|
346
|
-
tensors:
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
1519
|
+
rank: pipeline.adapter.rank,
|
|
1520
|
+
alpha: pipeline.adapter.alpha,
|
|
1521
|
+
targetModules: normalizeLoraTargetModules(pipeline.adapter),
|
|
1522
|
+
tensors: trainerOutput.tensors,
|
|
1523
|
+
weightsFormat: 'safetensors',
|
|
1524
|
+
weightsPath: weightsFilename,
|
|
1525
|
+
metadata: {
|
|
1526
|
+
runnerKind: 'causal_lm_text_generation',
|
|
1527
|
+
runnerKey: preflight.runnerKey,
|
|
1528
|
+
runnerId: trainerOutput.runnerId || trainerInfo.runnerId,
|
|
1529
|
+
trainerId: trainerOutput.trainerId,
|
|
1530
|
+
trainerSource: trainerInfo.source,
|
|
1531
|
+
datasetHash,
|
|
1532
|
+
workloadSha256: loadedWorkload.workloadSha256,
|
|
1533
|
+
metrics: trainerOutput.metrics,
|
|
1534
|
+
receipts: trainerOutput.receipts,
|
|
1535
|
+
},
|
|
350
1536
|
});
|
|
1537
|
+
if (!exported.weights) {
|
|
1538
|
+
throw new Error('Causal-LM LoRA safetensors export did not return weights bytes.');
|
|
1539
|
+
}
|
|
351
1540
|
const manifestPath = join(layout.exports, `${checkpointId}.adapter.manifest.json`);
|
|
1541
|
+
const runtimeManifestPath = join(layout.exports, 'runtime-adapter-manifest.json');
|
|
1542
|
+
const weightsPath = join(layout.exports, weightsFilename);
|
|
1543
|
+
await writeFile(weightsPath, new Uint8Array(exported.weights));
|
|
352
1544
|
await writeFile(manifestPath, exported.json, 'utf8');
|
|
353
|
-
await
|
|
1545
|
+
await writeFile(runtimeManifestPath, exported.json, 'utf8');
|
|
1546
|
+
await loadLoRAFromManifest(exported.manifest, {
|
|
1547
|
+
readFile: async (filePath) => readFile(join(layout.exports, filePath)),
|
|
1548
|
+
});
|
|
354
1549
|
const artifactPayload = {
|
|
355
1550
|
...buildArtifact(loadedWorkload, {
|
|
356
1551
|
prefix: 'lora_export',
|
|
@@ -361,6 +1556,17 @@ async function exportToyLoraModel(loadedWorkload, layout, model, checkpointId, c
|
|
|
361
1556
|
}),
|
|
362
1557
|
checkpointId,
|
|
363
1558
|
manifestPath,
|
|
1559
|
+
runtimeManifestPath,
|
|
1560
|
+
weightsPath,
|
|
1561
|
+
weightsSha256: exported.weightsSha256 || null,
|
|
1562
|
+
runnerKey: preflight.runnerKey,
|
|
1563
|
+
trainer: {
|
|
1564
|
+
runnerId: trainerOutput.runnerId || trainerInfo.runnerId,
|
|
1565
|
+
trainerId: trainerOutput.trainerId,
|
|
1566
|
+
source: trainerInfo.source,
|
|
1567
|
+
exportName: trainerInfo.exportName,
|
|
1568
|
+
},
|
|
1569
|
+
metrics: trainerOutput.metrics,
|
|
364
1570
|
manifest: exported.manifest,
|
|
365
1571
|
};
|
|
366
1572
|
const artifactFile = await writeJsonArtifact(
|
|
@@ -370,11 +1576,369 @@ async function exportToyLoraModel(loadedWorkload, layout, model, checkpointId, c
|
|
|
370
1576
|
return {
|
|
371
1577
|
checkpointId,
|
|
372
1578
|
manifestPath,
|
|
1579
|
+
runtimeManifestPath,
|
|
1580
|
+
weightsPath,
|
|
1581
|
+
weightsSha256: exported.weightsSha256 || null,
|
|
373
1582
|
exportPath: artifactFile.path,
|
|
374
1583
|
manifest: exported.manifest,
|
|
375
1584
|
};
|
|
376
1585
|
}
|
|
377
1586
|
|
|
1587
|
+
function hasExternalCausalLmTrainer(loadedWorkload, options = {}) {
|
|
1588
|
+
return typeof options.causalLmTrainer === 'function'
|
|
1589
|
+
|| Boolean(getPipelineConfig(loadedWorkload.workload).trainer);
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
function assertInternalCausalLmTrainerAllowed(workload, compatibility) {
|
|
1593
|
+
if (compatibility.observed.requiresExternalTrainer !== true) return;
|
|
1594
|
+
throw new Error(
|
|
1595
|
+
`Causal-LM LoRA base "${workload.baseModelId}" requires lora.trainer.modulePath or runLoraPipeline({ causalLmTrainer }) ` +
|
|
1596
|
+
'because the internal full-graph runner does not train packed q4k base weights.'
|
|
1597
|
+
);
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
async function runInternalCausalLmLoraPipeline(options, layout, compatibility) {
|
|
1601
|
+
const loadedWorkload = options.loadedWorkload;
|
|
1602
|
+
const workload = loadedWorkload.workload;
|
|
1603
|
+
const pipeline = getPipelineConfig(workload);
|
|
1604
|
+
const datasetPath = await resolveDatasetPathForLoadedWorkload(workload.datasetPath, loadedWorkload);
|
|
1605
|
+
const preflight = await preflightCausalLmLoraWorkload(workload, {
|
|
1606
|
+
datasetPath,
|
|
1607
|
+
fetch: options.fetch,
|
|
1608
|
+
readFile: options.readFile,
|
|
1609
|
+
});
|
|
1610
|
+
if (!preflight.supported || preflight.blockedReasons.length > 0) {
|
|
1611
|
+
throw new Error(`Causal-LM LoRA workload is blocked: ${preflight.blockedReasons.join(',')}`);
|
|
1612
|
+
}
|
|
1613
|
+
assertInternalCausalLmTrainerAllowed(workload, compatibility);
|
|
1614
|
+
if (Math.floor(Number(workload.training.batchSize)) !== 1) {
|
|
1615
|
+
throw new Error('Causal-LM LoRA workload requires training.batchSize=1.');
|
|
1616
|
+
}
|
|
1617
|
+
const freeze = getCausalLmFreezeConfig(workload);
|
|
1618
|
+
const fixture = await createCausalLmLoraFixture(workload);
|
|
1619
|
+
try {
|
|
1620
|
+
const dataset = await loadCausalLmTextPairSamples(workload, datasetPath, fixture.tokenizer);
|
|
1621
|
+
const evalReports = [];
|
|
1622
|
+
const checkpointArtifacts = [];
|
|
1623
|
+
const exports = [];
|
|
1624
|
+
const baselineEvalReports = await evaluateCausalLmLoraModel(workload, fixture, dataset, layout, {
|
|
1625
|
+
checkpointId: 'base-model',
|
|
1626
|
+
checkpointStep: 0,
|
|
1627
|
+
stage: 'base_model',
|
|
1628
|
+
configHash: workload.configHash,
|
|
1629
|
+
workloadPath: loadedWorkload.absolutePath,
|
|
1630
|
+
workloadSha256: loadedWorkload.workloadSha256,
|
|
1631
|
+
});
|
|
1632
|
+
for (const report of baselineEvalReports) {
|
|
1633
|
+
evalReports.push(report);
|
|
1634
|
+
await appendScoreboardRow(layout.scoreboard, {
|
|
1635
|
+
artifactType: 'training_scoreboard',
|
|
1636
|
+
schemaVersion: 1,
|
|
1637
|
+
generatedAt: new Date().toISOString(),
|
|
1638
|
+
checkpointId: report.checkpointId,
|
|
1639
|
+
checkpointStep: report.checkpointStep,
|
|
1640
|
+
stage: report.stage,
|
|
1641
|
+
evalDatasetId: report.evalDatasetId,
|
|
1642
|
+
primaryMetric: report.primaryMetric,
|
|
1643
|
+
primaryScore: report.primaryScore,
|
|
1644
|
+
loss: report.loss,
|
|
1645
|
+
metrics: {
|
|
1646
|
+
loss: report.loss,
|
|
1647
|
+
primaryScore: report.primaryScore,
|
|
1648
|
+
},
|
|
1649
|
+
}, {
|
|
1650
|
+
selectionMetric: workload.selectionMetric,
|
|
1651
|
+
selectionGoal: workload.selectionGoal,
|
|
1652
|
+
});
|
|
1653
|
+
}
|
|
1654
|
+
const baselineReportsByEvalDatasetId = Object.fromEntries(
|
|
1655
|
+
baselineEvalReports.map((report) => [report.evalDatasetId, report])
|
|
1656
|
+
);
|
|
1657
|
+
const finalizedCheckpointIds = new Set();
|
|
1658
|
+
const runner = new TrainingRunner(createLoraRunnerTrainingConfig(workload, freeze), {
|
|
1659
|
+
optimizer: createLoraOptimizer(workload),
|
|
1660
|
+
crossEntropyLoss,
|
|
1661
|
+
clipGradients,
|
|
1662
|
+
trainingObjective: createCausalLmTrainingObjective(),
|
|
1663
|
+
onCheckpoint: async (checkpoint) => {
|
|
1664
|
+
const checkpointId = `checkpoint-${String(checkpoint.step).padStart(6, '0')}`;
|
|
1665
|
+
if (finalizedCheckpointIds.has(checkpointId)) return;
|
|
1666
|
+
finalizedCheckpointIds.add(checkpointId);
|
|
1667
|
+
const checkpointPayload = {
|
|
1668
|
+
...buildArtifact(loadedWorkload, {
|
|
1669
|
+
prefix: 'lora_ckpt',
|
|
1670
|
+
id: checkpointId,
|
|
1671
|
+
artifactType: 'training_checkpoint',
|
|
1672
|
+
datasetPath: dataset.absolutePath,
|
|
1673
|
+
datasetHash: dataset.datasetHash,
|
|
1674
|
+
checkpointStep: checkpoint.step,
|
|
1675
|
+
}),
|
|
1676
|
+
checkpointId,
|
|
1677
|
+
checkpointPath: checkpoint.path,
|
|
1678
|
+
optimizerStatePresent: true,
|
|
1679
|
+
schedulerStatePresent: workload.training.optimizer.scheduler.enabled === true,
|
|
1680
|
+
runnerKey: compatibility.observed.runnerKey,
|
|
1681
|
+
baseModelRef: fixture.baseModelRef,
|
|
1682
|
+
tokenization: dataset.tokenization,
|
|
1683
|
+
resumeLineage: checkpoint.metadata?.lineage || null,
|
|
1684
|
+
};
|
|
1685
|
+
await writeJsonArtifact(
|
|
1686
|
+
join(layout.checkpoints, checkpointId, 'checkpoint.json'),
|
|
1687
|
+
checkpointPayload
|
|
1688
|
+
);
|
|
1689
|
+
const checkpointArtifact = await writeJsonArtifact(
|
|
1690
|
+
join(layout.checkpoints, checkpointId, 'checkpoint.complete.json'),
|
|
1691
|
+
checkpointPayload
|
|
1692
|
+
);
|
|
1693
|
+
checkpointArtifacts.push({
|
|
1694
|
+
checkpointId,
|
|
1695
|
+
checkpointPath: checkpoint.path,
|
|
1696
|
+
markerPath: checkpointArtifact.path,
|
|
1697
|
+
checkpointStep: checkpoint.step,
|
|
1698
|
+
});
|
|
1699
|
+
if (pipeline.export?.enabled === true && pipeline.export.atCheckpoints === true) {
|
|
1700
|
+
exports.push(await exportCausalLmLoraModel(
|
|
1701
|
+
loadedWorkload,
|
|
1702
|
+
layout,
|
|
1703
|
+
fixture,
|
|
1704
|
+
checkpointId,
|
|
1705
|
+
checkpoint.step,
|
|
1706
|
+
dataset.datasetHash
|
|
1707
|
+
));
|
|
1708
|
+
}
|
|
1709
|
+
const reports = await evaluateCausalLmLoraModel(workload, fixture, dataset, layout, {
|
|
1710
|
+
checkpointId,
|
|
1711
|
+
checkpointStep: checkpoint.step,
|
|
1712
|
+
configHash: workload.configHash,
|
|
1713
|
+
workloadPath: loadedWorkload.absolutePath,
|
|
1714
|
+
workloadSha256: loadedWorkload.workloadSha256,
|
|
1715
|
+
baselineReportsByEvalDatasetId,
|
|
1716
|
+
});
|
|
1717
|
+
for (const report of reports) {
|
|
1718
|
+
evalReports.push(report);
|
|
1719
|
+
await appendScoreboardRow(layout.scoreboard, {
|
|
1720
|
+
artifactType: 'training_scoreboard',
|
|
1721
|
+
schemaVersion: 1,
|
|
1722
|
+
generatedAt: new Date().toISOString(),
|
|
1723
|
+
checkpointId,
|
|
1724
|
+
checkpointStep: checkpoint.step,
|
|
1725
|
+
stage: report.stage,
|
|
1726
|
+
evalDatasetId: report.evalDatasetId,
|
|
1727
|
+
primaryMetric: report.primaryMetric,
|
|
1728
|
+
primaryScore: report.primaryScore,
|
|
1729
|
+
loss: report.loss,
|
|
1730
|
+
qualityClaim: report.qualityClaim || null,
|
|
1731
|
+
metrics: {
|
|
1732
|
+
loss: report.loss,
|
|
1733
|
+
primaryScore: report.primaryScore,
|
|
1734
|
+
},
|
|
1735
|
+
}, {
|
|
1736
|
+
selectionMetric: workload.selectionMetric,
|
|
1737
|
+
selectionGoal: workload.selectionGoal,
|
|
1738
|
+
});
|
|
1739
|
+
}
|
|
1740
|
+
},
|
|
1741
|
+
});
|
|
1742
|
+
const metrics = await runner.run(
|
|
1743
|
+
fixture.model,
|
|
1744
|
+
createCausalLmDatasetBatches(dataset.samples),
|
|
1745
|
+
{
|
|
1746
|
+
epochs: 1,
|
|
1747
|
+
batchSize: 1,
|
|
1748
|
+
shuffle: false,
|
|
1749
|
+
maxSteps: workload.training.steps,
|
|
1750
|
+
checkpointEvery: workload.checkpointEvery,
|
|
1751
|
+
checkpointKey: join(layout.checkpoints, 'latest.state.json'),
|
|
1752
|
+
modelId: workload.baseModelId,
|
|
1753
|
+
modelUrl: fixture.baseModelUrl,
|
|
1754
|
+
tokenizerHash: fixture.baseManifest?.tokenizerHash || null,
|
|
1755
|
+
}
|
|
1756
|
+
);
|
|
1757
|
+
const finalCheckpointId = runner.lastCheckpoint
|
|
1758
|
+
? `checkpoint-${String(runner.lastCheckpoint.step).padStart(6, '0')}`
|
|
1759
|
+
: null;
|
|
1760
|
+
if (
|
|
1761
|
+
pipeline.export?.enabled !== false
|
|
1762
|
+
&& finalCheckpointId
|
|
1763
|
+
&& exports.every((entry) => entry.checkpointId !== finalCheckpointId)
|
|
1764
|
+
) {
|
|
1765
|
+
exports.push(await exportCausalLmLoraModel(
|
|
1766
|
+
loadedWorkload,
|
|
1767
|
+
layout,
|
|
1768
|
+
fixture,
|
|
1769
|
+
finalCheckpointId,
|
|
1770
|
+
runner.lastCheckpoint.step,
|
|
1771
|
+
dataset.datasetHash
|
|
1772
|
+
));
|
|
1773
|
+
}
|
|
1774
|
+
return {
|
|
1775
|
+
ok: true,
|
|
1776
|
+
kind: 'lora',
|
|
1777
|
+
action: 'run',
|
|
1778
|
+
runnerKind: 'causal_lm_lora',
|
|
1779
|
+
workloadId: workload.id,
|
|
1780
|
+
runRoot: layout.runRoot,
|
|
1781
|
+
preflight,
|
|
1782
|
+
checkpointArtifacts,
|
|
1783
|
+
evalReports,
|
|
1784
|
+
exports,
|
|
1785
|
+
metrics,
|
|
1786
|
+
lastCheckpoint: runner.lastCheckpoint,
|
|
1787
|
+
dataset: {
|
|
1788
|
+
path: dataset.absolutePath,
|
|
1789
|
+
rowCount: dataset.rowCount,
|
|
1790
|
+
sampleCount: dataset.samples.length,
|
|
1791
|
+
datasetHash: dataset.datasetHash,
|
|
1792
|
+
tokenization: dataset.tokenization,
|
|
1793
|
+
},
|
|
1794
|
+
baseModel: {
|
|
1795
|
+
id: workload.baseModelId,
|
|
1796
|
+
ref: fixture.baseModelRef,
|
|
1797
|
+
url: fixture.baseModelUrl,
|
|
1798
|
+
},
|
|
1799
|
+
};
|
|
1800
|
+
} finally {
|
|
1801
|
+
await fixture.cleanup();
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
async function runProviderCausalLmLoraPipeline(options, compatibility) {
|
|
1806
|
+
const loadedWorkload = options.loadedWorkload;
|
|
1807
|
+
const workload = loadedWorkload.workload;
|
|
1808
|
+
const pipeline = getPipelineConfig(workload);
|
|
1809
|
+
if (!workload.datasetPath) {
|
|
1810
|
+
throw new Error('preflightCausalLmLoraWorkload requires workload.datasetPath.');
|
|
1811
|
+
}
|
|
1812
|
+
const layout = await createLoraRunLayout(options, workload);
|
|
1813
|
+
await writeRunContract(layout, buildRunContract(loadedWorkload));
|
|
1814
|
+
await writeWorkloadLock(layout, loadedWorkload);
|
|
1815
|
+
const datasetPath = await resolveDatasetPathForLoadedWorkload(workload.datasetPath, loadedWorkload);
|
|
1816
|
+
const dataset = await loadTextPairsDataset(datasetPath, {
|
|
1817
|
+
fetch: options.fetch,
|
|
1818
|
+
readFile: options.readFile,
|
|
1819
|
+
});
|
|
1820
|
+
if (dataset.rowCount < 1) {
|
|
1821
|
+
throw new Error(`Causal-LM LoRA dataset ${dataset.absolutePath} has no rows.`);
|
|
1822
|
+
}
|
|
1823
|
+
const datasetHash = hashTextPairsDataset(dataset);
|
|
1824
|
+
const preflight = await preflightCausalLmLoraWorkload(workload, {
|
|
1825
|
+
datasetPath: dataset.absolutePath,
|
|
1826
|
+
fetch: options.fetch,
|
|
1827
|
+
readFile: options.readFile,
|
|
1828
|
+
});
|
|
1829
|
+
const trainerInfo = await resolveCausalLmTrainer(loadedWorkload, options);
|
|
1830
|
+
const trainerResult = await trainerInfo.train({
|
|
1831
|
+
schemaVersion: 1,
|
|
1832
|
+
runnerKind: 'causal_lm_lora',
|
|
1833
|
+
workload,
|
|
1834
|
+
loadedWorkload,
|
|
1835
|
+
compatibility,
|
|
1836
|
+
preflight,
|
|
1837
|
+
dataset: {
|
|
1838
|
+
absolutePath: dataset.absolutePath,
|
|
1839
|
+
rowCount: dataset.rowCount,
|
|
1840
|
+
rows: dataset.rows,
|
|
1841
|
+
datasetHash,
|
|
1842
|
+
},
|
|
1843
|
+
adapter: pipeline.adapter,
|
|
1844
|
+
training: workload.training,
|
|
1845
|
+
export: pipeline.export,
|
|
1846
|
+
layout,
|
|
1847
|
+
});
|
|
1848
|
+
const trainerOutput = normalizeCausalLmTrainerOutput(trainerResult, workload);
|
|
1849
|
+
const checkpointPayload = {
|
|
1850
|
+
...buildArtifact(loadedWorkload, {
|
|
1851
|
+
prefix: 'lora_ckpt',
|
|
1852
|
+
id: trainerOutput.checkpointId,
|
|
1853
|
+
artifactType: 'training_checkpoint',
|
|
1854
|
+
datasetPath: dataset.absolutePath,
|
|
1855
|
+
datasetHash,
|
|
1856
|
+
checkpointStep: trainerOutput.checkpointStep,
|
|
1857
|
+
}),
|
|
1858
|
+
checkpointId: trainerOutput.checkpointId,
|
|
1859
|
+
checkpointPath: join(layout.checkpoints, trainerOutput.checkpointId, 'trainer-output.json'),
|
|
1860
|
+
optimizerStatePresent: false,
|
|
1861
|
+
schedulerStatePresent: false,
|
|
1862
|
+
runnerKey: preflight.runnerKey,
|
|
1863
|
+
trainer: {
|
|
1864
|
+
runnerId: trainerOutput.runnerId || trainerInfo.runnerId,
|
|
1865
|
+
trainerId: trainerOutput.trainerId,
|
|
1866
|
+
source: trainerInfo.source,
|
|
1867
|
+
exportName: trainerInfo.exportName,
|
|
1868
|
+
},
|
|
1869
|
+
tensorNames: trainerOutput.tensors.map((entry) => entry.name),
|
|
1870
|
+
metrics: trainerOutput.metrics,
|
|
1871
|
+
receipts: trainerOutput.receipts,
|
|
1872
|
+
};
|
|
1873
|
+
await writeJsonArtifact(
|
|
1874
|
+
join(layout.checkpoints, trainerOutput.checkpointId, 'trainer-output.json'),
|
|
1875
|
+
checkpointPayload
|
|
1876
|
+
);
|
|
1877
|
+
const checkpointArtifact = await writeJsonArtifact(
|
|
1878
|
+
join(layout.checkpoints, trainerOutput.checkpointId, 'checkpoint.complete.json'),
|
|
1879
|
+
checkpointPayload
|
|
1880
|
+
);
|
|
1881
|
+
const exported = pipeline.export?.enabled === false
|
|
1882
|
+
? null
|
|
1883
|
+
: await exportProviderCausalLmLoraModel(
|
|
1884
|
+
loadedWorkload,
|
|
1885
|
+
layout,
|
|
1886
|
+
trainerOutput,
|
|
1887
|
+
trainerOutput.checkpointId,
|
|
1888
|
+
trainerOutput.checkpointStep,
|
|
1889
|
+
datasetHash,
|
|
1890
|
+
trainerInfo,
|
|
1891
|
+
preflight
|
|
1892
|
+
);
|
|
1893
|
+
const evalReports = await writeProviderEvalReports({
|
|
1894
|
+
loadedWorkload,
|
|
1895
|
+
workload,
|
|
1896
|
+
layout,
|
|
1897
|
+
dataset,
|
|
1898
|
+
datasetHash,
|
|
1899
|
+
preflight,
|
|
1900
|
+
trainerOutput,
|
|
1901
|
+
checkpointId: trainerOutput.checkpointId,
|
|
1902
|
+
checkpointStep: trainerOutput.checkpointStep,
|
|
1903
|
+
});
|
|
1904
|
+
return {
|
|
1905
|
+
ok: true,
|
|
1906
|
+
kind: 'lora',
|
|
1907
|
+
action: 'run',
|
|
1908
|
+
runnerKind: 'causal_lm_lora',
|
|
1909
|
+
workloadId: workload.id,
|
|
1910
|
+
runRoot: layout.runRoot,
|
|
1911
|
+
preflight,
|
|
1912
|
+
checkpointArtifacts: [{
|
|
1913
|
+
checkpointId: trainerOutput.checkpointId,
|
|
1914
|
+
checkpointPath: checkpointPayload.checkpointPath,
|
|
1915
|
+
markerPath: checkpointArtifact.path,
|
|
1916
|
+
checkpointStep: trainerOutput.checkpointStep,
|
|
1917
|
+
}],
|
|
1918
|
+
evalReports,
|
|
1919
|
+
exports: exported ? [exported] : [],
|
|
1920
|
+
metrics: trainerOutput.metrics,
|
|
1921
|
+
lastCheckpoint: {
|
|
1922
|
+
id: trainerOutput.checkpointId,
|
|
1923
|
+
step: trainerOutput.checkpointStep,
|
|
1924
|
+
path: checkpointPayload.checkpointPath,
|
|
1925
|
+
},
|
|
1926
|
+
};
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1929
|
+
async function runCausalLmLoraPipeline(options, compatibility) {
|
|
1930
|
+
if (!options.loadedWorkload.workload.datasetPath) {
|
|
1931
|
+
throw new Error('preflightCausalLmLoraWorkload requires workload.datasetPath.');
|
|
1932
|
+
}
|
|
1933
|
+
if (hasExternalCausalLmTrainer(options.loadedWorkload, options)) {
|
|
1934
|
+
return runProviderCausalLmLoraPipeline(options, compatibility);
|
|
1935
|
+
}
|
|
1936
|
+
const layout = await createLoraRunLayout(options, options.loadedWorkload.workload);
|
|
1937
|
+
await writeRunContract(layout, buildRunContract(options.loadedWorkload));
|
|
1938
|
+
await writeWorkloadLock(layout, options.loadedWorkload);
|
|
1939
|
+
return runInternalCausalLmLoraPipeline(options, layout, compatibility);
|
|
1940
|
+
}
|
|
1941
|
+
|
|
378
1942
|
async function selectLatestCheckpoint(runRoot) {
|
|
379
1943
|
const checkpointsDir = join(runRoot, 'checkpoints');
|
|
380
1944
|
const entries = await readdir(checkpointsDir, { withFileTypes: true });
|
|
@@ -399,29 +1963,14 @@ export async function runLoraPipeline(options) {
|
|
|
399
1963
|
if (workload.kind !== 'lora') {
|
|
400
1964
|
throw new Error('runLoraPipeline requires a lora workload.');
|
|
401
1965
|
}
|
|
402
|
-
|
|
403
|
-
|
|
1966
|
+
const compatibility = getLoraRunnerCompatibility(workload);
|
|
1967
|
+
if (!compatibility.supported) {
|
|
1968
|
+
assertLoraRunnerCompatibility(workload);
|
|
404
1969
|
}
|
|
405
|
-
if (workload
|
|
406
|
-
|
|
1970
|
+
if (isCausalLmLoraWorkload(workload, compatibility)) {
|
|
1971
|
+
return runCausalLmLoraPipeline(options, compatibility);
|
|
407
1972
|
}
|
|
408
|
-
const layout = options
|
|
409
|
-
? {
|
|
410
|
-
runRoot: resolve(String(options.runRoot)),
|
|
411
|
-
logs: join(resolve(String(options.runRoot)), 'logs'),
|
|
412
|
-
checkpoints: join(resolve(String(options.runRoot)), 'checkpoints'),
|
|
413
|
-
eval: join(resolve(String(options.runRoot)), 'eval'),
|
|
414
|
-
scoreboard: join(resolve(String(options.runRoot)), 'scoreboard'),
|
|
415
|
-
exports: join(resolve(String(options.runRoot)), 'exports'),
|
|
416
|
-
compare: join(resolve(String(options.runRoot)), 'compare'),
|
|
417
|
-
qualityGate: join(resolve(String(options.runRoot)), 'quality-gate'),
|
|
418
|
-
}
|
|
419
|
-
: await createTrainingRunLayout({
|
|
420
|
-
kind: 'lora',
|
|
421
|
-
workloadId: workload.id,
|
|
422
|
-
timestamp: options.timestamp || null,
|
|
423
|
-
});
|
|
424
|
-
await Promise.all(Object.values(layout).map((dirPath) => mkdir(dirPath, { recursive: true })));
|
|
1973
|
+
const layout = await createLoraRunLayout(options, workload);
|
|
425
1974
|
await writeRunContract(layout, buildRunContract(loadedWorkload));
|
|
426
1975
|
await writeWorkloadLock(layout, loadedWorkload);
|
|
427
1976
|
const dataset = await loadToyLoraDataset(workload.datasetPath);
|
|
@@ -430,6 +1979,7 @@ export async function runLoraPipeline(options) {
|
|
|
430
1979
|
const evalReports = [];
|
|
431
1980
|
const checkpointArtifacts = [];
|
|
432
1981
|
const exports = [];
|
|
1982
|
+
const finalizedCheckpointIds = new Set();
|
|
433
1983
|
const runner = new TrainingRunner({
|
|
434
1984
|
training: {
|
|
435
1985
|
enabled: true,
|
|
@@ -513,6 +2063,8 @@ export async function runLoraPipeline(options) {
|
|
|
513
2063
|
},
|
|
514
2064
|
onCheckpoint: async (checkpoint) => {
|
|
515
2065
|
const checkpointId = `checkpoint-${String(checkpoint.step).padStart(6, '0')}`;
|
|
2066
|
+
if (finalizedCheckpointIds.has(checkpointId)) return;
|
|
2067
|
+
finalizedCheckpointIds.add(checkpointId);
|
|
516
2068
|
const checkpointPayload = {
|
|
517
2069
|
...buildArtifact(loadedWorkload, {
|
|
518
2070
|
prefix: 'lora_ckpt',
|
|
@@ -709,45 +2261,138 @@ export async function watchLoraCheckpoints(options) {
|
|
|
709
2261
|
});
|
|
710
2262
|
}
|
|
711
2263
|
|
|
712
|
-
|
|
713
|
-
const
|
|
714
|
-
|
|
715
|
-
|
|
2264
|
+
async function listJsonFilesRecursive(rootDir) {
|
|
2265
|
+
const results = [];
|
|
2266
|
+
let entries = [];
|
|
2267
|
+
try {
|
|
2268
|
+
entries = await readdir(rootDir, { withFileTypes: true });
|
|
2269
|
+
} catch (error) {
|
|
2270
|
+
if (error?.code === 'ENOENT') return results;
|
|
2271
|
+
throw error;
|
|
2272
|
+
}
|
|
716
2273
|
for (const entry of entries) {
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
2274
|
+
const absolutePath = join(rootDir, entry.name);
|
|
2275
|
+
if (entry.isDirectory()) {
|
|
2276
|
+
results.push(...await listJsonFilesRecursive(absolutePath));
|
|
2277
|
+
continue;
|
|
2278
|
+
}
|
|
2279
|
+
if (entry.isFile() && entry.name.endsWith('.json')) {
|
|
2280
|
+
results.push(absolutePath);
|
|
2281
|
+
}
|
|
2282
|
+
}
|
|
2283
|
+
return results.sort((left, right) => left.localeCompare(right));
|
|
2284
|
+
}
|
|
2285
|
+
|
|
2286
|
+
function resolveComparableReportMetric(report, metric) {
|
|
2287
|
+
if (!report || typeof report !== 'object') return null;
|
|
2288
|
+
const direct = report[metric];
|
|
2289
|
+
if (typeof direct === 'number' && Number.isFinite(direct)) {
|
|
2290
|
+
return direct;
|
|
2291
|
+
}
|
|
2292
|
+
const nested = report.metrics?.[metric];
|
|
2293
|
+
if (typeof nested === 'number' && Number.isFinite(nested)) {
|
|
2294
|
+
return nested;
|
|
2295
|
+
}
|
|
2296
|
+
if (nested && typeof nested === 'object' && typeof nested.score === 'number' && Number.isFinite(nested.score)) {
|
|
2297
|
+
return nested.score;
|
|
720
2298
|
}
|
|
721
|
-
|
|
2299
|
+
return null;
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
async function loadRunWorkloadForComparison(runRoot) {
|
|
2303
|
+
try {
|
|
2304
|
+
const raw = await readFile(join(runRoot, 'workload.lock.json'), 'utf8');
|
|
2305
|
+
const lock = JSON.parse(raw);
|
|
2306
|
+
return lock.workload || null;
|
|
2307
|
+
} catch (error) {
|
|
2308
|
+
if (error?.code === 'ENOENT') return null;
|
|
2309
|
+
throw error;
|
|
2310
|
+
}
|
|
2311
|
+
}
|
|
2312
|
+
|
|
2313
|
+
function sortTrainingReports(reports, workload = null) {
|
|
2314
|
+
const metric = String(workload?.selectionMetric || reports[0]?.primaryMetric || 'primaryScore').trim();
|
|
2315
|
+
const goal = String(workload?.selectionGoal || (metric === 'loss' ? 'min' : 'max')).trim();
|
|
2316
|
+
return reports
|
|
722
2317
|
.slice()
|
|
723
2318
|
.sort((left, right) => {
|
|
724
|
-
const
|
|
725
|
-
const
|
|
2319
|
+
const missingScore = goal === 'min' ? Number.POSITIVE_INFINITY : Number.NEGATIVE_INFINITY;
|
|
2320
|
+
const leftScore = resolveComparableReportMetric(left, metric) ?? missingScore;
|
|
2321
|
+
const rightScore = resolveComparableReportMetric(right, metric) ?? missingScore;
|
|
2322
|
+
if (goal === 'min') return leftScore - rightScore;
|
|
726
2323
|
return rightScore - leftScore;
|
|
727
2324
|
});
|
|
2325
|
+
}
|
|
2326
|
+
|
|
2327
|
+
async function loadTrainingEvalReports(rootDir) {
|
|
2328
|
+
const files = await listJsonFilesRecursive(rootDir);
|
|
2329
|
+
const reports = [];
|
|
2330
|
+
for (const filePath of files) {
|
|
2331
|
+
const raw = await readFile(filePath, 'utf8');
|
|
2332
|
+
const report = JSON.parse(raw);
|
|
2333
|
+
if (report?.artifactType === 'training_eval_report') {
|
|
2334
|
+
reports.push({
|
|
2335
|
+
...report,
|
|
2336
|
+
reportPath: report.reportPath || filePath,
|
|
2337
|
+
});
|
|
2338
|
+
}
|
|
2339
|
+
}
|
|
2340
|
+
return reports;
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
export async function compareLoraRun(options) {
|
|
2344
|
+
const runRoot = resolve(String(options.runRoot));
|
|
2345
|
+
const workload = await loadRunWorkloadForComparison(runRoot);
|
|
2346
|
+
const sorted = sortTrainingReports(await loadTrainingEvalReports(join(runRoot, 'eval')), workload);
|
|
728
2347
|
const payload = {
|
|
729
2348
|
artifactType: 'training_compare_report',
|
|
730
2349
|
schemaVersion: 1,
|
|
731
2350
|
generatedAt: new Date().toISOString(),
|
|
732
|
-
runRoot
|
|
2351
|
+
runRoot,
|
|
2352
|
+
selectionMetric: workload?.selectionMetric || sorted[0]?.primaryMetric || null,
|
|
2353
|
+
selectionGoal: workload?.selectionGoal || null,
|
|
733
2354
|
count: sorted.length,
|
|
734
2355
|
best: sorted[0] || null,
|
|
735
2356
|
reports: sorted.map((report) => ({
|
|
2357
|
+
stage: report.stage || null,
|
|
736
2358
|
checkpointId: report.checkpointId || null,
|
|
737
2359
|
evalDatasetId: report.evalDatasetId || null,
|
|
738
2360
|
primaryMetric: report.primaryMetric || null,
|
|
739
2361
|
primaryScore: report.primaryScore ?? null,
|
|
2362
|
+
loss: report.loss ?? null,
|
|
2363
|
+
baseline: report.baseline || null,
|
|
2364
|
+
qualityClaim: report.qualityClaim || null,
|
|
740
2365
|
accuracy: report.accuracy ?? null,
|
|
2366
|
+
agentEval: report.agentEval || report.heldoutGate || null,
|
|
741
2367
|
reportPath: report.reportPath || null,
|
|
742
2368
|
})),
|
|
743
2369
|
};
|
|
744
|
-
const artifact = await writeJsonArtifact(join(
|
|
2370
|
+
const artifact = await writeJsonArtifact(join(runRoot, 'compare', 'compare.json'), payload);
|
|
745
2371
|
return {
|
|
746
2372
|
...payload,
|
|
747
2373
|
comparePath: artifact.path,
|
|
748
2374
|
};
|
|
749
2375
|
}
|
|
750
2376
|
|
|
2377
|
+
function collectRequiredImprovementEvalIds(workload) {
|
|
2378
|
+
const evalDatasets = Array.isArray(workload?.evalDatasets) ? workload.evalDatasets : [];
|
|
2379
|
+
return evalDatasets
|
|
2380
|
+
.filter((entry) => entry?.quality?.requireImprovement === true)
|
|
2381
|
+
.map((entry) => entry.id);
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
function summarizeQualityClaims(reports) {
|
|
2385
|
+
const claims = reports
|
|
2386
|
+
.map((report) => report.qualityClaim)
|
|
2387
|
+
.filter((claim) => claim && typeof claim === 'object');
|
|
2388
|
+
return {
|
|
2389
|
+
count: claims.length,
|
|
2390
|
+
improvedCount: claims.filter((claim) => claim.improved === true).length,
|
|
2391
|
+
requiredCount: claims.filter((claim) => claim.requireImprovement === true).length,
|
|
2392
|
+
failedRequiredCount: claims.filter((claim) => claim.requireImprovement === true && claim.improved !== true).length,
|
|
2393
|
+
};
|
|
2394
|
+
}
|
|
2395
|
+
|
|
751
2396
|
export async function qualityGateLoraRun(options) {
|
|
752
2397
|
const runRoot = resolve(String(options.runRoot));
|
|
753
2398
|
const requiredPaths = [
|
|
@@ -763,6 +2408,47 @@ export async function qualityGateLoraRun(options) {
|
|
|
763
2408
|
checks.push({ path: filePath, ok: false, error: error?.message || String(error) });
|
|
764
2409
|
}
|
|
765
2410
|
}
|
|
2411
|
+
const workload = await loadRunWorkloadForComparison(runRoot);
|
|
2412
|
+
const evalReports = await loadTrainingEvalReports(join(runRoot, 'eval'));
|
|
2413
|
+
const qualitySummary = summarizeQualityClaims(evalReports);
|
|
2414
|
+
const agentEvalSummary = summarizeAgentEvalReportRequirements(workload, evalReports);
|
|
2415
|
+
const requiredImprovementEvalIds = collectRequiredImprovementEvalIds(workload);
|
|
2416
|
+
if (evalReports.length > 0) {
|
|
2417
|
+
checks.push({
|
|
2418
|
+
name: 'eval_reports',
|
|
2419
|
+
path: join(runRoot, 'eval'),
|
|
2420
|
+
ok: true,
|
|
2421
|
+
count: evalReports.length,
|
|
2422
|
+
});
|
|
2423
|
+
}
|
|
2424
|
+
if (qualitySummary.count > 0) {
|
|
2425
|
+
checks.push({
|
|
2426
|
+
name: 'baseline_quality_claims',
|
|
2427
|
+
path: join(runRoot, 'eval'),
|
|
2428
|
+
ok: qualitySummary.failedRequiredCount === 0,
|
|
2429
|
+
...qualitySummary,
|
|
2430
|
+
});
|
|
2431
|
+
}
|
|
2432
|
+
if (requiredImprovementEvalIds.length > 0 && qualitySummary.count === 0) {
|
|
2433
|
+
checks.push({
|
|
2434
|
+
name: 'required_improvement_claims',
|
|
2435
|
+
path: join(runRoot, 'eval'),
|
|
2436
|
+
ok: false,
|
|
2437
|
+
requiredEvalDatasetIds: requiredImprovementEvalIds,
|
|
2438
|
+
error: 'No baseline quality claims were written for eval datasets that require improvement.',
|
|
2439
|
+
});
|
|
2440
|
+
}
|
|
2441
|
+
if (agentEvalSummary.requiredCount > 0) {
|
|
2442
|
+
checks.push({
|
|
2443
|
+
name: 'agent_heldout_eval',
|
|
2444
|
+
path: join(runRoot, 'eval'),
|
|
2445
|
+
ok: agentEvalSummary.failedCount === 0,
|
|
2446
|
+
...agentEvalSummary,
|
|
2447
|
+
error: agentEvalSummary.failedCount === 0
|
|
2448
|
+
? null
|
|
2449
|
+
: 'One or more required agent held-out eval gates are missing or failing.',
|
|
2450
|
+
});
|
|
2451
|
+
}
|
|
766
2452
|
const passed = checks.every((entry) => entry.ok === true);
|
|
767
2453
|
const payload = {
|
|
768
2454
|
artifactType: 'training_quality_gate',
|
|
@@ -770,6 +2456,8 @@ export async function qualityGateLoraRun(options) {
|
|
|
770
2456
|
generatedAt: new Date().toISOString(),
|
|
771
2457
|
runRoot,
|
|
772
2458
|
passed,
|
|
2459
|
+
qualitySummary,
|
|
2460
|
+
agentEvalSummary,
|
|
773
2461
|
checks,
|
|
774
2462
|
};
|
|
775
2463
|
const artifact = await writeJsonArtifact(join(runRoot, 'quality-gate', 'quality-gate.json'), payload);
|