@tuned-tensor/cli 0.10.0 → 0.12.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/{chunk-DAP2Z7EZ.js → chunk-HDGBDHOB.js} +474 -74
- package/dist/chunk-HDGBDHOB.js.map +1 -0
- package/dist/index.js +235 -84
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +52 -19
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/architecture.md +8 -2
- package/docs/local-runtime/spark.md +39 -2
- package/package.json +5 -1
- package/training/local-runner/pyproject.toml +2 -2
- package/training/local-runner/src/evaluate.py +44 -18
- package/training/local-runner/src/model_contract.py +134 -14
- package/training/local-runner/src/prefetch.py +25 -8
- package/training/local-runner/src/serve.py +14 -6
- package/training/local-runner/src/sft_data.py +7 -3
- package/training/local-runner/src/train.py +48 -14
- package/training/local-runner/uv.lock +54 -54
- package/dist/chunk-DAP2Z7EZ.js.map +0 -1
|
@@ -558,17 +558,59 @@ async function assertArtifactManifest(manifestPath, options = {}) {
|
|
|
558
558
|
import { lstat as lstat2, readdir as readdir2 } from "fs/promises";
|
|
559
559
|
import { resolve as resolve2 } from "path";
|
|
560
560
|
import { fileURLToPath } from "url";
|
|
561
|
+
var NEMOTRON_BF16_REVISION = "ce38b6ab8b252b4b8ee7165b4605e93191cafd73";
|
|
562
|
+
var MUSE_GLIMMER_BF16_REVISION = "a4e59da52a7bc87ae7251dd5545c0dd437c44b68";
|
|
561
563
|
var TRAINING_MODELS = [
|
|
562
564
|
{
|
|
563
565
|
id: "Qwen/Qwen3.5-2B",
|
|
564
566
|
family: "qwen3_5",
|
|
567
|
+
modelLoader: "causal_lm",
|
|
565
568
|
defaultLearningRate: 1e-5,
|
|
566
569
|
defaultPerDeviceBatchSize: 1,
|
|
567
570
|
defaultGradientAccumulationSteps: 8,
|
|
568
571
|
defaultLoraRank: 16,
|
|
569
572
|
defaultLoraAlpha: 32,
|
|
570
573
|
defaultLoraDropout: 0.05,
|
|
571
|
-
defaultMaxSeqLength: 2048
|
|
574
|
+
defaultMaxSeqLength: 2048,
|
|
575
|
+
loraTargetModules: "all-linear",
|
|
576
|
+
gradientCheckpointing: false
|
|
577
|
+
},
|
|
578
|
+
{
|
|
579
|
+
id: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
|
|
580
|
+
family: "nemotron_h",
|
|
581
|
+
defaultRevision: NEMOTRON_BF16_REVISION,
|
|
582
|
+
modelLoader: "causal_lm",
|
|
583
|
+
defaultLearningRate: 1e-5,
|
|
584
|
+
defaultPerDeviceBatchSize: 1,
|
|
585
|
+
defaultGradientAccumulationSteps: 8,
|
|
586
|
+
defaultLoraRank: 16,
|
|
587
|
+
defaultLoraAlpha: 32,
|
|
588
|
+
defaultLoraDropout: 0.05,
|
|
589
|
+
defaultMaxSeqLength: 1024,
|
|
590
|
+
// Avoid one LoRA pair for every routed expert matrix. Adapt the shared
|
|
591
|
+
// attention (q/k/v/o) and Mamba input projections instead, keeping the
|
|
592
|
+
// adapter bounded. out_proj/conv1d are rejected by PEFT for Mamba models.
|
|
593
|
+
loraTargetModules: ["q_proj", "k_proj", "v_proj", "o_proj", "in_proj"],
|
|
594
|
+
gradientCheckpointing: true
|
|
595
|
+
},
|
|
596
|
+
{
|
|
597
|
+
id: "meta-models/Muse-Glimmer-30B",
|
|
598
|
+
family: "muse_glimmer",
|
|
599
|
+
defaultRevision: MUSE_GLIMMER_BF16_REVISION,
|
|
600
|
+
// Muse Glimmer is a vision-language model whose text tower is not exposed
|
|
601
|
+
// through the causal-LM auto mapping, so the bundled runtime loads the full
|
|
602
|
+
// conditional-generation checkpoint (vision tower frozen and unused) via
|
|
603
|
+
// AutoModelForImageTextToText for text-only SFT.
|
|
604
|
+
modelLoader: "image_text_to_text",
|
|
605
|
+
defaultLearningRate: 1e-5,
|
|
606
|
+
defaultPerDeviceBatchSize: 1,
|
|
607
|
+
defaultGradientAccumulationSteps: 8,
|
|
608
|
+
defaultLoraRank: 16,
|
|
609
|
+
defaultLoraAlpha: 32,
|
|
610
|
+
defaultLoraDropout: 0.05,
|
|
611
|
+
defaultMaxSeqLength: 2048,
|
|
612
|
+
loraTargetModules: "all-linear",
|
|
613
|
+
gradientCheckpointing: true
|
|
572
614
|
}
|
|
573
615
|
];
|
|
574
616
|
function resolveTrainingModel(modelId) {
|
|
@@ -578,7 +620,7 @@ function resolveTrainingModel(modelId) {
|
|
|
578
620
|
);
|
|
579
621
|
if (!model) {
|
|
580
622
|
throw new Error(
|
|
581
|
-
`Unsupported base model "${modelId}". Supported
|
|
623
|
+
`Unsupported base model "${modelId}". Supported models: ${TRAINING_MODELS.map((item) => item.id).join(", ")}`
|
|
582
624
|
);
|
|
583
625
|
}
|
|
584
626
|
return model;
|
|
@@ -586,7 +628,22 @@ function resolveTrainingModel(modelId) {
|
|
|
586
628
|
function canonicalizeTrainingModel(modelId) {
|
|
587
629
|
return resolveTrainingModel(modelId).id;
|
|
588
630
|
}
|
|
589
|
-
|
|
631
|
+
function resolveRequestedBaseModelRevision(modelId, requestedRevision) {
|
|
632
|
+
const model = resolveTrainingModel(modelId);
|
|
633
|
+
if (model.defaultRevision && requestedRevision && requestedRevision !== model.defaultRevision) {
|
|
634
|
+
throw new Error(
|
|
635
|
+
`${model.id} must use certified revision ${model.defaultRevision}; got ${requestedRevision}.`
|
|
636
|
+
);
|
|
637
|
+
}
|
|
638
|
+
return requestedRevision ?? model.defaultRevision;
|
|
639
|
+
}
|
|
640
|
+
function defaultBaseModelRevision(modelId) {
|
|
641
|
+
return resolveRequestedBaseModelRevision(modelId);
|
|
642
|
+
}
|
|
643
|
+
function resolveModelLoader(modelId) {
|
|
644
|
+
return resolveTrainingModel(modelId).modelLoader;
|
|
645
|
+
}
|
|
646
|
+
var CERTIFIED_QWEN_TEXT_CONFIG = {
|
|
590
647
|
model_type: "qwen3_5_text",
|
|
591
648
|
hidden_size: 2048,
|
|
592
649
|
num_hidden_layers: 24,
|
|
@@ -595,27 +652,99 @@ var CERTIFIED_TEXT_CONFIG = {
|
|
|
595
652
|
intermediate_size: 6144,
|
|
596
653
|
vocab_size: 248320
|
|
597
654
|
};
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
655
|
+
var CERTIFIED_MUSE_GLIMMER_TEXT_CONFIG = {
|
|
656
|
+
model_type: "muse_glimmer_text",
|
|
657
|
+
hidden_size: 6656,
|
|
658
|
+
num_hidden_layers: 52,
|
|
659
|
+
num_attention_heads: 32,
|
|
660
|
+
num_key_value_heads: 2,
|
|
661
|
+
intermediate_size: 19968,
|
|
662
|
+
vocab_size: 202048
|
|
663
|
+
};
|
|
664
|
+
var CERTIFIED_NEMOTRON_CONFIG = {
|
|
665
|
+
hidden_size: 2688,
|
|
666
|
+
num_hidden_layers: 52,
|
|
667
|
+
num_attention_heads: 32,
|
|
668
|
+
num_key_value_heads: 2,
|
|
669
|
+
intermediate_size: 1856,
|
|
670
|
+
vocab_size: 131072,
|
|
671
|
+
n_routed_experts: 128,
|
|
672
|
+
num_experts_per_tok: 6,
|
|
673
|
+
num_nextn_predict_layers: 1,
|
|
674
|
+
max_position_embeddings: 262144
|
|
675
|
+
};
|
|
676
|
+
function isRecord(value) {
|
|
677
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
678
|
+
}
|
|
679
|
+
function hasArchitecture(config, architecture) {
|
|
603
680
|
const architectures = config.architectures;
|
|
681
|
+
return Array.isArray(architectures) && architectures.includes(architecture);
|
|
682
|
+
}
|
|
683
|
+
function assertCertifiedTextTowerConfig(config, label, certifiedModelId, architecture, certifiedTextConfig) {
|
|
684
|
+
if (!hasArchitecture(config, architecture)) {
|
|
685
|
+
throw new Error(`${label} is not the certified ${certifiedModelId} architecture.`);
|
|
686
|
+
}
|
|
604
687
|
const textConfig = config.text_config;
|
|
605
|
-
if (
|
|
606
|
-
throw new Error(
|
|
607
|
-
`${label} is not the certified Qwen/Qwen3.5-2B architecture.`
|
|
608
|
-
);
|
|
688
|
+
if (!isRecord(textConfig)) {
|
|
689
|
+
throw new Error(`${label} is not the certified ${certifiedModelId} architecture.`);
|
|
609
690
|
}
|
|
610
|
-
const
|
|
611
|
-
|
|
612
|
-
if (text[key] !== expected) {
|
|
691
|
+
for (const [key, expected] of Object.entries(certifiedTextConfig)) {
|
|
692
|
+
if (textConfig[key] !== expected) {
|
|
613
693
|
throw new Error(
|
|
614
|
-
`${label} is not the certified
|
|
694
|
+
`${label} is not the certified ${certifiedModelId} architecture: text_config.${key} must be ${JSON.stringify(expected)}.`
|
|
615
695
|
);
|
|
616
696
|
}
|
|
617
697
|
}
|
|
618
698
|
}
|
|
699
|
+
function assertCertifiedBaseModelConfig(value, label = "base-model config.json", expectedModelId) {
|
|
700
|
+
if (!isRecord(value)) {
|
|
701
|
+
throw new Error(`${label} must contain a JSON object.`);
|
|
702
|
+
}
|
|
703
|
+
const config = value;
|
|
704
|
+
const expectedFamily = expectedModelId ? resolveTrainingModel(expectedModelId).family : void 0;
|
|
705
|
+
if (config.model_type === "qwen3_5") {
|
|
706
|
+
if (expectedFamily && expectedFamily !== "qwen3_5") {
|
|
707
|
+
throw new Error(`${label} does not match requested base model ${expectedModelId}.`);
|
|
708
|
+
}
|
|
709
|
+
assertCertifiedTextTowerConfig(
|
|
710
|
+
config,
|
|
711
|
+
label,
|
|
712
|
+
"Qwen/Qwen3.5-2B",
|
|
713
|
+
"Qwen3_5ForConditionalGeneration",
|
|
714
|
+
CERTIFIED_QWEN_TEXT_CONFIG
|
|
715
|
+
);
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
718
|
+
if (config.model_type === "muse_glimmer") {
|
|
719
|
+
if (expectedFamily && expectedFamily !== "muse_glimmer") {
|
|
720
|
+
throw new Error(`${label} does not match requested base model ${expectedModelId}.`);
|
|
721
|
+
}
|
|
722
|
+
assertCertifiedTextTowerConfig(
|
|
723
|
+
config,
|
|
724
|
+
label,
|
|
725
|
+
"meta-models/Muse-Glimmer-30B",
|
|
726
|
+
"MuseGlimmerForConditionalGeneration",
|
|
727
|
+
CERTIFIED_MUSE_GLIMMER_TEXT_CONFIG
|
|
728
|
+
);
|
|
729
|
+
return;
|
|
730
|
+
}
|
|
731
|
+
if (config.model_type === "nemotron_h" && hasArchitecture(config, "NemotronHForCausalLM")) {
|
|
732
|
+
if (expectedFamily && expectedFamily !== "nemotron_h") {
|
|
733
|
+
throw new Error(`${label} does not match requested base model ${expectedModelId}.`);
|
|
734
|
+
}
|
|
735
|
+
for (const [key, expected] of Object.entries(CERTIFIED_NEMOTRON_CONFIG)) {
|
|
736
|
+
if (config[key] !== expected) {
|
|
737
|
+
throw new Error(
|
|
738
|
+
`${label} is not the certified nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 architecture: ${key} must be ${JSON.stringify(expected)}.`
|
|
739
|
+
);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
return;
|
|
743
|
+
}
|
|
744
|
+
throw new Error(
|
|
745
|
+
`${label} is not a certified TT Local base-model architecture.`
|
|
746
|
+
);
|
|
747
|
+
}
|
|
619
748
|
function localModelArtifactPath(uri) {
|
|
620
749
|
if (uri.startsWith("file://")) return fileURLToPath(new URL(uri));
|
|
621
750
|
if (/^[a-z][a-z0-9+.-]*:/i.test(uri)) {
|
|
@@ -1743,8 +1872,10 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1743
1872
|
}
|
|
1744
1873
|
|
|
1745
1874
|
// src/local-runtime/prefetch.ts
|
|
1746
|
-
import {
|
|
1747
|
-
import {
|
|
1875
|
+
import { createHash as createHash4 } from "crypto";
|
|
1876
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
1877
|
+
import { lstat as lstat3, mkdir as mkdir5, readFile as readFile5, readdir as readdir4, realpath as realpath2, stat as stat4 } from "fs/promises";
|
|
1878
|
+
import { isAbsolute as isAbsolute3, join as join5, relative as relative2, resolve as resolve8, sep as sep2 } from "path";
|
|
1748
1879
|
|
|
1749
1880
|
// src/local-runtime/process-runner.ts
|
|
1750
1881
|
import { createHash as createHash3 } from "crypto";
|
|
@@ -2024,11 +2155,195 @@ function withOfflineHuggingFaceCacheEnvironment(env, modelCache) {
|
|
|
2024
2155
|
}
|
|
2025
2156
|
|
|
2026
2157
|
// src/local-runtime/prefetch.ts
|
|
2158
|
+
var NEMOTRON_BF16_FILE_DIGESTS = {
|
|
2159
|
+
".gitattributes": { algorithm: "sha1", digest: "fe3f694ce14b577c9833fca6e83c5e26a7119127" },
|
|
2160
|
+
"LICENSE": { algorithm: "sha1", digest: "4d76cc87c9edb280753e3bee3b499ae461113835" },
|
|
2161
|
+
"README.md": { algorithm: "sha1", digest: "8ec43ee9f3ee345202e4a6ef752c51b2619f7425" },
|
|
2162
|
+
"accuracy_plot.png": { algorithm: "sha256", digest: "1397995f8a5d34d819a069d86c02914990e9ecfc2fce101e233924da9cf5ddcf" },
|
|
2163
|
+
"agentic_coding_benchmarks.png": { algorithm: "sha256", digest: "06cf5486ae94ecd7cc6f91ae3b4bae21c7435253a294cb89f54f7ec42eda8847" },
|
|
2164
|
+
"bias.md": { algorithm: "sha1", digest: "cdcc8055f965222fabddabcabf22885d3a8bacc1" },
|
|
2165
|
+
"chat_template.jinja": { algorithm: "sha1", digest: "d85b0c772f8fe585063847c5f6bf5ec48eb210be" },
|
|
2166
|
+
"config.json": { algorithm: "sha1", digest: "993b612989e6aa67da45f908ea5983cb777e7678" },
|
|
2167
|
+
"explainability.md": { algorithm: "sha1", digest: "2435f23c865e7daba6a6eceef2f214b86df75d24" },
|
|
2168
|
+
"generation_config.json": { algorithm: "sha1", digest: "a41201df53f9d0769947989353b5d02e70491c56" },
|
|
2169
|
+
"model.safetensors.index.json": { algorithm: "sha1", digest: "3cd409443b1896d9adc5c3981af301841f85329a" },
|
|
2170
|
+
"special_tokens_map.json": { algorithm: "sha1", digest: "0451f37912edb7f8a4bf04d77eef13f6130f0515" },
|
|
2171
|
+
"tokenizer.json": { algorithm: "sha256", digest: "623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7" },
|
|
2172
|
+
"tokenizer_config.json": { algorithm: "sha1", digest: "c96e5ad00986bfa0b666d4bc425666f1c51d6d1b" },
|
|
2173
|
+
"model-00001-of-00014.safetensors": { algorithm: "sha256", digest: "7a3e74d24969eac6657cb9b6422091fd062b7d30fd4bc90ecee6c3d7f692626a" },
|
|
2174
|
+
"model-00002-of-00014.safetensors": { algorithm: "sha256", digest: "a80c3846e959f6dda34aa031cbb668b766643691ac7e5c9f43f62e6fb1221059" },
|
|
2175
|
+
"model-00003-of-00014.safetensors": { algorithm: "sha256", digest: "996bf3a46668e14075cb82259390267c2b1f71446caeb3a5f1f9e0bd7a079ae7" },
|
|
2176
|
+
"model-00004-of-00014.safetensors": { algorithm: "sha256", digest: "cce1d79e7ff2e2b746b9d97166156aa1dcec796116675cc7fd1aa70a8d73ccb2" },
|
|
2177
|
+
"model-00005-of-00014.safetensors": { algorithm: "sha256", digest: "9338f59d3fd5f074288c71a9559452d5b5c581da00e9ccb9fe7aa2ce9b04d2ad" },
|
|
2178
|
+
"model-00006-of-00014.safetensors": { algorithm: "sha256", digest: "e947fa4097c68263ba3d097fceb32a300a22cebd9d909fb5198af7d18a4022fe" },
|
|
2179
|
+
"model-00007-of-00014.safetensors": { algorithm: "sha256", digest: "c7362630bf4256275ca58354ed8f31506cec67ffab30573e753703fd5ae48795" },
|
|
2180
|
+
"model-00008-of-00014.safetensors": { algorithm: "sha256", digest: "1b33a5a6b5505b25b57c90ddda19e9cd1598176468917df0c56116fa6f6072cb" },
|
|
2181
|
+
"model-00009-of-00014.safetensors": { algorithm: "sha256", digest: "fcc9b48caff427beaf1f87b78cf4ab42bc2a3c43b3c487bccb013ba22d7468b2" },
|
|
2182
|
+
"model-00010-of-00014.safetensors": { algorithm: "sha256", digest: "d1924fd54d1b77a15aff8df591a317e8a82a58e46d6f5742fa21e90e0be3cfa6" },
|
|
2183
|
+
"model-00011-of-00014.safetensors": { algorithm: "sha256", digest: "e9d0b83bcf04bd4d5758b94e6495a3f1595d121d82faffaa27ce6a884e2c8757" },
|
|
2184
|
+
"model-00012-of-00014.safetensors": { algorithm: "sha256", digest: "fce184f112cb0b97024cadc60f993f6f660e365d04e21c240fb3c75747bd3a01" },
|
|
2185
|
+
"model-00013-of-00014.safetensors": { algorithm: "sha256", digest: "f427d037d2a3e4d44323325aa38571e3551e8ad702f469c3dacc4e59ea2e50d1" },
|
|
2186
|
+
"model-00014-of-00014.safetensors": { algorithm: "sha256", digest: "64577b275ca4e7e5266eae0903674f7f46ec2a8cbf4f4f1a3207f80d503cd1d0" },
|
|
2187
|
+
"privacy.md": { algorithm: "sha1", digest: "e3bf30aa1c42a488298ef37383647dc99e1cd39e" },
|
|
2188
|
+
"safety.md": { algorithm: "sha1", digest: "f53bf94a4635862eed5cf3e0bc799b93f81d20ac" }
|
|
2189
|
+
};
|
|
2190
|
+
var MUSE_GLIMMER_FILE_DIGESTS = {
|
|
2191
|
+
".gitattributes": { algorithm: "sha1", digest: "52373fe24473b1aa44333d318f578ae6bf04b49b" },
|
|
2192
|
+
"LICENSE": { algorithm: "sha1", digest: "d645695673349e3947e8e5ae42332d0ac3164cd7" },
|
|
2193
|
+
"README.md": { algorithm: "sha1", digest: "a25f87c1c1fbd88f367a552fd14427baa8cb947e" },
|
|
2194
|
+
"USAGE_POLICY.md": { algorithm: "sha1", digest: "1a9ed6cffc54585daf9cc904c07cb6e436a9444b" },
|
|
2195
|
+
"chat_template.jinja": { algorithm: "sha1", digest: "7507f3c9f38809152732c045df3977848f3916a6" },
|
|
2196
|
+
"config.json": { algorithm: "sha1", digest: "190826dc834c13c86b8dc68e775a888a852d7c34" },
|
|
2197
|
+
"generation_config.json": { algorithm: "sha1", digest: "b69a50a4239f42707d54a479e0143b8ee56c8bcc" },
|
|
2198
|
+
"model.safetensors.index.json": { algorithm: "sha1", digest: "00b257ef0c8bce1b001a6266f6b1f66f4fe16a6b" },
|
|
2199
|
+
"processor_config.json": { algorithm: "sha1", digest: "ec9a07be13ea1dadf7395131871753b13f406c26" },
|
|
2200
|
+
"tokenizer_config.json": { algorithm: "sha1", digest: "fe7f0bb90fb0d0288e3a505974ae0e4baf1d3e61" },
|
|
2201
|
+
"tokenizer.json": { algorithm: "sha256", digest: "c9dbee66967b58f31a7c27f723c3760da3526ccd0427578e8905b0abb0031c4d" },
|
|
2202
|
+
"model-00001-of-00002.safetensors": { algorithm: "sha256", digest: "8eef61530e1283642c77ce2e6721feb5c6f348fa055c00e90f2844a136372694" },
|
|
2203
|
+
"model-00002-of-00002.safetensors": { algorithm: "sha256", digest: "b58cc2144ba1ba1af4420f67f4ca3ced7f09298510b80464cc75018a0be14381" }
|
|
2204
|
+
};
|
|
2205
|
+
function certifiedFileDigests(modelId) {
|
|
2206
|
+
switch (modelId) {
|
|
2207
|
+
case "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16":
|
|
2208
|
+
return NEMOTRON_BF16_FILE_DIGESTS;
|
|
2209
|
+
case "meta-models/Muse-Glimmer-30B":
|
|
2210
|
+
return MUSE_GLIMMER_FILE_DIGESTS;
|
|
2211
|
+
default:
|
|
2212
|
+
return void 0;
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2215
|
+
async function certifiedFileDigest(path, expected) {
|
|
2216
|
+
const metadata = await stat4(path);
|
|
2217
|
+
const digest = createHash4(expected.algorithm);
|
|
2218
|
+
if (expected.algorithm === "sha1") digest.update(`blob ${metadata.size}\0`);
|
|
2219
|
+
for await (const chunk of createReadStream2(path)) digest.update(chunk);
|
|
2220
|
+
return digest.digest("hex");
|
|
2221
|
+
}
|
|
2222
|
+
function isStrictPhysicalDescendant(parent, child) {
|
|
2223
|
+
const candidate = relative2(parent, child);
|
|
2224
|
+
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep2}`) && !isAbsolute3(candidate);
|
|
2225
|
+
}
|
|
2226
|
+
async function verifySnapshotCacheLocation(path, modelId, revision, modelCache) {
|
|
2227
|
+
const repository = `models--${modelId.replaceAll("/", "--")}`;
|
|
2228
|
+
const layout = resolveHuggingFaceCacheLayout(modelCache);
|
|
2229
|
+
const repositoryRoot = join5(layout.hubCache, repository);
|
|
2230
|
+
const expectedPath = join5(repositoryRoot, "snapshots", revision);
|
|
2231
|
+
const [
|
|
2232
|
+
physicalCacheRoot,
|
|
2233
|
+
physicalHubRoot,
|
|
2234
|
+
physicalRepositoryRoot,
|
|
2235
|
+
physicalBlobRoot,
|
|
2236
|
+
physicalPath,
|
|
2237
|
+
physicalExpectedPath
|
|
2238
|
+
] = await Promise.all([
|
|
2239
|
+
realpath2(layout.hfHome).catch(() => null),
|
|
2240
|
+
realpath2(layout.hubCache).catch(() => null),
|
|
2241
|
+
realpath2(repositoryRoot).catch(() => null),
|
|
2242
|
+
realpath2(join5(repositoryRoot, "blobs")).catch(() => null),
|
|
2243
|
+
realpath2(path).catch(() => null),
|
|
2244
|
+
realpath2(expectedPath).catch(() => null)
|
|
2245
|
+
]);
|
|
2246
|
+
if (!physicalCacheRoot || !physicalHubRoot || !physicalRepositoryRoot || !physicalBlobRoot || !physicalPath || !physicalExpectedPath || physicalPath !== physicalExpectedPath) {
|
|
2247
|
+
throw new Error(
|
|
2248
|
+
`${modelId} must use the certified Hugging Face cache snapshot ${expectedPath}; got ${path}.`
|
|
2249
|
+
);
|
|
2250
|
+
}
|
|
2251
|
+
if (!isStrictPhysicalDescendant(physicalCacheRoot, physicalHubRoot)) {
|
|
2252
|
+
throw new Error(`Hugging Face hub cache escapes its configured cache root: ${layout.hubCache}`);
|
|
2253
|
+
}
|
|
2254
|
+
if (!isStrictPhysicalDescendant(physicalHubRoot, physicalRepositoryRoot)) {
|
|
2255
|
+
throw new Error(`Hugging Face repository escapes its configured hub cache: ${repositoryRoot}`);
|
|
2256
|
+
}
|
|
2257
|
+
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalBlobRoot)) {
|
|
2258
|
+
throw new Error(`Hugging Face blob store escapes its repository: ${join5(repositoryRoot, "blobs")}`);
|
|
2259
|
+
}
|
|
2260
|
+
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalExpectedPath)) {
|
|
2261
|
+
throw new Error(`Hugging Face snapshot escapes its configured repository: ${expectedPath}`);
|
|
2262
|
+
}
|
|
2263
|
+
return { repositoryRoot, physicalBlobRoot };
|
|
2264
|
+
}
|
|
2265
|
+
async function requireReportedCachePath(label, reported, expected) {
|
|
2266
|
+
const [physicalReported, physicalExpected] = await Promise.all([
|
|
2267
|
+
reported ? realpath2(reported).catch(() => null) : null,
|
|
2268
|
+
realpath2(expected).catch(() => null)
|
|
2269
|
+
]);
|
|
2270
|
+
if (!physicalReported || !physicalExpected || physicalReported !== physicalExpected) {
|
|
2271
|
+
throw new Error(`Model prefetch returned an unexpected ${label}: ${String(reported)}.`);
|
|
2272
|
+
}
|
|
2273
|
+
}
|
|
2274
|
+
async function verifyModelPrefetchCacheReport(args) {
|
|
2275
|
+
if (!args.output.snapshot_path) {
|
|
2276
|
+
throw new Error("Model prefetch did not return a snapshot path.");
|
|
2277
|
+
}
|
|
2278
|
+
const effectiveHubCache = resolveHuggingFaceCacheLayout(args.modelCache).hubCache;
|
|
2279
|
+
await Promise.all([
|
|
2280
|
+
requireReportedCachePath("model cache", args.output.model_cache, args.modelCache),
|
|
2281
|
+
requireReportedCachePath("HF home", args.output.hf_home, args.modelCache),
|
|
2282
|
+
requireReportedCachePath("hub cache", args.output.hub_cache, effectiveHubCache)
|
|
2283
|
+
]);
|
|
2284
|
+
await verifySnapshotCacheLocation(
|
|
2285
|
+
args.output.snapshot_path,
|
|
2286
|
+
args.modelId,
|
|
2287
|
+
args.revision,
|
|
2288
|
+
args.modelCache
|
|
2289
|
+
);
|
|
2290
|
+
}
|
|
2291
|
+
async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelCache) {
|
|
2292
|
+
const revision = defaultBaseModelRevision(modelId);
|
|
2293
|
+
if (!revision) return;
|
|
2294
|
+
const digests = certifiedFileDigests(modelId);
|
|
2295
|
+
if (!digests) {
|
|
2296
|
+
throw new Error(`No certified file digests are registered for pinned model ${modelId}.`);
|
|
2297
|
+
}
|
|
2298
|
+
if (!modelCache) {
|
|
2299
|
+
throw new Error(`A certified local snapshot for ${modelId} requires paths.modelCache.`);
|
|
2300
|
+
}
|
|
2301
|
+
const { physicalBlobRoot } = await verifySnapshotCacheLocation(
|
|
2302
|
+
path,
|
|
2303
|
+
modelId,
|
|
2304
|
+
revision,
|
|
2305
|
+
modelCache
|
|
2306
|
+
);
|
|
2307
|
+
const physicalFiles = /* @__PURE__ */ new Map();
|
|
2308
|
+
for (const file of snapshotFiles) {
|
|
2309
|
+
const metadata = await lstat3(file);
|
|
2310
|
+
const target = await realpath2(file);
|
|
2311
|
+
if (metadata.isSymbolicLink()) {
|
|
2312
|
+
if (!isStrictPhysicalDescendant(physicalBlobRoot, target)) {
|
|
2313
|
+
throw new Error(`Certified snapshot symbolic link escapes its Hugging Face blob store: ${file}`);
|
|
2314
|
+
}
|
|
2315
|
+
}
|
|
2316
|
+
physicalFiles.set(file, target);
|
|
2317
|
+
}
|
|
2318
|
+
const actualFiles = new Set(snapshotFiles.map((file) => relative2(path, file).split("\\").join("/")));
|
|
2319
|
+
const expectedFiles = new Set(Object.keys(digests));
|
|
2320
|
+
const unexpected = [...actualFiles].filter((name) => !expectedFiles.has(name)).sort();
|
|
2321
|
+
if (unexpected.length > 0) {
|
|
2322
|
+
throw new Error(`Certified snapshot contains unexpected file: ${join5(path, unexpected[0])}`);
|
|
2323
|
+
}
|
|
2324
|
+
for (const [name, expected] of Object.entries(digests)) {
|
|
2325
|
+
if (!actualFiles.has(name)) {
|
|
2326
|
+
throw new Error(`Certified snapshot file is missing: ${join5(path, name)}`);
|
|
2327
|
+
}
|
|
2328
|
+
const file = join5(path, name);
|
|
2329
|
+
const physicalFile = physicalFiles.get(file);
|
|
2330
|
+
if (await certifiedFileDigest(physicalFile, expected) !== expected.digest) {
|
|
2331
|
+
throw new Error(`Certified snapshot file checksum mismatch: ${file}`);
|
|
2332
|
+
}
|
|
2333
|
+
if (await realpath2(file) !== physicalFile) {
|
|
2334
|
+
throw new Error(`Certified snapshot file changed during verification: ${file}`);
|
|
2335
|
+
}
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
2027
2338
|
function buildModelPrefetchPayload(request, config) {
|
|
2028
2339
|
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
2340
|
+
const revision = resolveRequestedBaseModelRevision(
|
|
2341
|
+
model.id,
|
|
2342
|
+
request.hyperparameters.base_model_revision
|
|
2343
|
+
);
|
|
2029
2344
|
return {
|
|
2030
2345
|
base_model: model.id,
|
|
2031
|
-
...
|
|
2346
|
+
...revision ? { revision } : {},
|
|
2032
2347
|
...config.paths.modelCache ? { model_cache: resolve8(config.paths.modelCache) } : {}
|
|
2033
2348
|
};
|
|
2034
2349
|
}
|
|
@@ -2039,7 +2354,7 @@ function transformersWeightName(name) {
|
|
|
2039
2354
|
const lower = name.split(/[\\/]/).at(-1)?.toLowerCase() ?? "";
|
|
2040
2355
|
return lower.endsWith(".safetensors") && !lower.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(lower);
|
|
2041
2356
|
}
|
|
2042
|
-
async function verifyLocalBaseModel(path) {
|
|
2357
|
+
async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
2043
2358
|
const root = await lstat3(path).catch(() => null);
|
|
2044
2359
|
if (!root) throw new Error(`paths.baseModel is set to ${path}, but that path does not exist.`);
|
|
2045
2360
|
if (root.isSymbolicLink()) {
|
|
@@ -2054,14 +2369,19 @@ async function verifyLocalBaseModel(path) {
|
|
|
2054
2369
|
const visit = async (directory) => {
|
|
2055
2370
|
for (const entry of await readdir4(directory, { withFileTypes: true })) {
|
|
2056
2371
|
const child = join5(directory, entry.name);
|
|
2372
|
+
if (entry.isSymbolicLink()) {
|
|
2373
|
+
const metadata2 = await stat4(child);
|
|
2374
|
+
if (!metadata2.isFile()) {
|
|
2375
|
+
throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
|
|
2376
|
+
}
|
|
2377
|
+
files.push({ path: child, size: metadata2.size });
|
|
2378
|
+
continue;
|
|
2379
|
+
}
|
|
2057
2380
|
if (entry.isDirectory()) {
|
|
2058
2381
|
await visit(child);
|
|
2059
2382
|
continue;
|
|
2060
2383
|
}
|
|
2061
2384
|
const metadata = await stat4(child);
|
|
2062
|
-
if (entry.isSymbolicLink() && !metadata.isFile()) {
|
|
2063
|
-
throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
|
|
2064
|
-
}
|
|
2065
2385
|
if (metadata.isFile()) files.push({ path: child, size: metadata.size });
|
|
2066
2386
|
}
|
|
2067
2387
|
};
|
|
@@ -2078,9 +2398,25 @@ async function verifyLocalBaseModel(path) {
|
|
|
2078
2398
|
if (!await required("config.json")) throw new Error(`Local base-model directory is missing config.json: ${path}`);
|
|
2079
2399
|
try {
|
|
2080
2400
|
const config = JSON.parse(await readFile5(join5(path, "config.json"), "utf8"));
|
|
2081
|
-
assertCertifiedBaseModelConfig(
|
|
2401
|
+
assertCertifiedBaseModelConfig(
|
|
2402
|
+
config,
|
|
2403
|
+
`Local base-model config ${join5(path, "config.json")}`,
|
|
2404
|
+
expectedModelId
|
|
2405
|
+
);
|
|
2082
2406
|
} catch (error) {
|
|
2083
|
-
|
|
2407
|
+
const detail = error instanceof Error ? `: ${error.message}` : "";
|
|
2408
|
+
throw new Error(
|
|
2409
|
+
`Local base-model directory has an invalid or unsupported config.json: ${path}${detail}`,
|
|
2410
|
+
{ cause: error }
|
|
2411
|
+
);
|
|
2412
|
+
}
|
|
2413
|
+
if (expectedModelId) {
|
|
2414
|
+
await verifyCertifiedLocalSnapshot(
|
|
2415
|
+
path,
|
|
2416
|
+
expectedModelId,
|
|
2417
|
+
files.map((file) => file.path),
|
|
2418
|
+
modelCache
|
|
2419
|
+
);
|
|
2084
2420
|
}
|
|
2085
2421
|
const vocabNames = [
|
|
2086
2422
|
"tokenizer.json",
|
|
@@ -2110,6 +2446,10 @@ async function verifyLocalBaseModel(path) {
|
|
|
2110
2446
|
};
|
|
2111
2447
|
}
|
|
2112
2448
|
async function prefetchBaseModel(args) {
|
|
2449
|
+
const payload = {
|
|
2450
|
+
...buildModelPrefetchPayload(args.request, args.config),
|
|
2451
|
+
...args.localOnly ? { local_files_only: true } : {}
|
|
2452
|
+
};
|
|
2113
2453
|
const artifactDir = resolve8(
|
|
2114
2454
|
args.config.artifactRoot,
|
|
2115
2455
|
args.localOnly ? "verify-base" : "prefetch",
|
|
@@ -2118,7 +2458,11 @@ async function prefetchBaseModel(args) {
|
|
|
2118
2458
|
await mkdir5(artifactDir, { recursive: true });
|
|
2119
2459
|
if (args.config.paths.baseModel) {
|
|
2120
2460
|
const localPath = resolve8(args.config.paths.baseModel);
|
|
2121
|
-
const verified = await verifyLocalBaseModel(
|
|
2461
|
+
const verified = await verifyLocalBaseModel(
|
|
2462
|
+
localPath,
|
|
2463
|
+
args.request.spec_snapshot.base_model,
|
|
2464
|
+
args.config.paths.modelCache
|
|
2465
|
+
);
|
|
2122
2466
|
return {
|
|
2123
2467
|
ok: true,
|
|
2124
2468
|
status: args.localOnly ? "completed" : "skipped",
|
|
@@ -2130,10 +2474,6 @@ async function prefetchBaseModel(args) {
|
|
|
2130
2474
|
reason: args.localOnly ? "Verified the configured local base-model artifact; no network access was used." : "paths.baseModel points at a verified local base-model artifact; no Hugging Face download is needed."
|
|
2131
2475
|
};
|
|
2132
2476
|
}
|
|
2133
|
-
const payload = {
|
|
2134
|
-
...buildModelPrefetchPayload(args.request, args.config),
|
|
2135
|
-
...args.localOnly ? { local_files_only: true } : {}
|
|
2136
|
-
};
|
|
2137
2477
|
const inputPath = join5(artifactDir, "prefetch-input.json");
|
|
2138
2478
|
const outputPath = join5(artifactDir, "prefetch-output.json");
|
|
2139
2479
|
const logPath = join5(artifactDir, "prefetch.log");
|
|
@@ -2153,12 +2493,13 @@ async function prefetchBaseModel(args) {
|
|
|
2153
2493
|
log_path: logPath
|
|
2154
2494
|
}
|
|
2155
2495
|
});
|
|
2156
|
-
const
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
payload.model_cache
|
|
2160
|
-
)
|
|
2496
|
+
const cacheEnvironment = withHuggingFaceCacheEnvironment(
|
|
2497
|
+
minimalMachineLearningEnvironment(process.env),
|
|
2498
|
+
payload.model_cache
|
|
2161
2499
|
);
|
|
2500
|
+
const effectiveModelCache = cacheEnvironment.HF_HOME;
|
|
2501
|
+
const effectiveHubCache = resolveHuggingFaceCacheLayout(effectiveModelCache).hubCache;
|
|
2502
|
+
const env = withBundledPythonEnvironment(cacheEnvironment);
|
|
2162
2503
|
const { exitCode } = await runLoggedProcess({
|
|
2163
2504
|
command: entrypoint.command,
|
|
2164
2505
|
commandArgs: entrypoint.commandArgs,
|
|
@@ -2179,14 +2520,35 @@ async function prefetchBaseModel(args) {
|
|
|
2179
2520
|
"Model prefetch did not resolve the base model to a 40-character immutable commit SHA."
|
|
2180
2521
|
);
|
|
2181
2522
|
}
|
|
2523
|
+
if (output.base_model !== payload.base_model) {
|
|
2524
|
+
throw new Error(`Model prefetch returned ${String(output.base_model)} for requested model ${payload.base_model}.`);
|
|
2525
|
+
}
|
|
2526
|
+
if (payload.revision && snapshotRevision.data !== payload.revision.toLowerCase()) {
|
|
2527
|
+
throw new Error(
|
|
2528
|
+
`Model prefetch resolved a different revision (${snapshotRevision.data}) than requested (${payload.revision}).`
|
|
2529
|
+
);
|
|
2530
|
+
}
|
|
2531
|
+
await verifyModelPrefetchCacheReport({
|
|
2532
|
+
modelId: payload.base_model,
|
|
2533
|
+
revision: snapshotRevision.data,
|
|
2534
|
+
modelCache: effectiveModelCache,
|
|
2535
|
+
output
|
|
2536
|
+
});
|
|
2537
|
+
if (defaultBaseModelRevision(payload.base_model)) {
|
|
2538
|
+
await verifyLocalBaseModel(
|
|
2539
|
+
output.snapshot_path,
|
|
2540
|
+
payload.base_model,
|
|
2541
|
+
effectiveModelCache
|
|
2542
|
+
);
|
|
2543
|
+
}
|
|
2182
2544
|
await args.reporter?.onEvent?.({
|
|
2183
2545
|
stage: "model_prefetch",
|
|
2184
2546
|
status: "completed",
|
|
2185
2547
|
message: "Base model is available in the local Hugging Face cache.",
|
|
2186
2548
|
details: {
|
|
2187
2549
|
base_model: output.base_model ?? payload.base_model,
|
|
2188
|
-
hf_home:
|
|
2189
|
-
hub_cache:
|
|
2550
|
+
hf_home: effectiveModelCache,
|
|
2551
|
+
hub_cache: effectiveHubCache,
|
|
2190
2552
|
snapshot_path: output.snapshot_path ?? null,
|
|
2191
2553
|
snapshot_revision: snapshotRevision.data,
|
|
2192
2554
|
file_count: output.file_count ?? null,
|
|
@@ -2198,9 +2560,9 @@ async function prefetchBaseModel(args) {
|
|
|
2198
2560
|
ok: true,
|
|
2199
2561
|
status: "completed",
|
|
2200
2562
|
base_model: output.base_model ?? payload.base_model,
|
|
2201
|
-
model_cache:
|
|
2202
|
-
hf_home:
|
|
2203
|
-
hub_cache:
|
|
2563
|
+
model_cache: effectiveModelCache,
|
|
2564
|
+
hf_home: effectiveModelCache,
|
|
2565
|
+
hub_cache: effectiveHubCache,
|
|
2204
2566
|
snapshot_path: output.snapshot_path,
|
|
2205
2567
|
snapshot_revision: snapshotRevision.data,
|
|
2206
2568
|
file_count: output.file_count,
|
|
@@ -2235,14 +2597,14 @@ function evaluateGeneralRegressionGate(comparison, policy) {
|
|
|
2235
2597
|
|
|
2236
2598
|
// src/local-runtime/orchestrator.ts
|
|
2237
2599
|
import { lstat as lstat4, mkdir as mkdir8, open as open2, readFile as readFile8, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
|
|
2238
|
-
import { createReadStream as
|
|
2239
|
-
import { createHash as
|
|
2600
|
+
import { createReadStream as createReadStream3 } from "fs";
|
|
2601
|
+
import { createHash as createHash6, randomUUID as randomUUID4 } from "crypto";
|
|
2240
2602
|
import { performance } from "perf_hooks";
|
|
2241
2603
|
import { homedir as homedir5 } from "os";
|
|
2242
|
-
import { basename, dirname as dirname8, join as join8, relative as
|
|
2604
|
+
import { basename, dirname as dirname8, join as join8, relative as relative3, resolve as resolve11 } from "path";
|
|
2243
2605
|
|
|
2244
2606
|
// src/local-runtime/evaluation.ts
|
|
2245
|
-
import { createHash as
|
|
2607
|
+
import { createHash as createHash5 } from "crypto";
|
|
2246
2608
|
import { mkdir as mkdir6, readFile as readFile6, rm as rm2, writeFile as writeFile4 } from "fs/promises";
|
|
2247
2609
|
import { dirname as dirname7, join as join6, resolve as resolve9 } from "path";
|
|
2248
2610
|
function normalize(value) {
|
|
@@ -2470,8 +2832,9 @@ async function runTransformersInference(args) {
|
|
|
2470
2832
|
kind: args.kind,
|
|
2471
2833
|
model_id: args.modelId,
|
|
2472
2834
|
base_model: args.baseModelId,
|
|
2835
|
+
model_source: args.modelSource,
|
|
2473
2836
|
base_model_revision: args.baseModelRevision,
|
|
2474
|
-
model_loader:
|
|
2837
|
+
model_loader: resolveModelLoader(args.baseModelId),
|
|
2475
2838
|
adapter_path: fileUriToPath(args.adapterPath),
|
|
2476
2839
|
system: args.system,
|
|
2477
2840
|
examples: args.examples.map((example, index) => ({
|
|
@@ -2544,7 +2907,7 @@ function assertEvaluationScoringReady(config) {
|
|
|
2544
2907
|
}
|
|
2545
2908
|
}
|
|
2546
2909
|
function baselineCacheKey(args) {
|
|
2547
|
-
return
|
|
2910
|
+
return createHash5("sha256").update(JSON.stringify({
|
|
2548
2911
|
v: 6,
|
|
2549
2912
|
inference_protocol_version: INFERENCE_PROTOCOL_VERSION,
|
|
2550
2913
|
package_version: args.packageVersion,
|
|
@@ -2642,19 +3005,26 @@ async function scoreGenerated(args) {
|
|
|
2642
3005
|
}
|
|
2643
3006
|
async function evaluateExamples(args) {
|
|
2644
3007
|
assertEvaluationScoringReady(args.config);
|
|
3008
|
+
const baseModelId = args.baseModelId ?? args.modelId;
|
|
3009
|
+
const requestedBaseModelRevision = resolveRequestedBaseModelRevision(
|
|
3010
|
+
baseModelId,
|
|
3011
|
+
args.baseModelRevision
|
|
3012
|
+
);
|
|
3013
|
+
const localModelSource = args.modelSource ? resolve9(args.modelSource) !== resolve9(baseModelId) : false;
|
|
3014
|
+
const baseModelRevision = localModelSource ? defaultBaseModelRevision(baseModelId) : requestedBaseModelRevision;
|
|
2645
3015
|
const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
|
|
2646
3016
|
const truncated = args.examples.length > maxExamples;
|
|
2647
3017
|
const seed = args.config.evaluation.sampleSeed ?? args.sampleSeed ?? 0;
|
|
2648
3018
|
const examples = truncated ? sampleExamples(args.examples, maxExamples, seed) : args.examples;
|
|
2649
3019
|
const evalSampleSeed = truncated || args.evalSplit === "spec_holdout" ? seed : null;
|
|
2650
3020
|
const cacheEligible = args.kind === "baseline" && args.config.evaluation.baselineCache && !args.config.dryRun && baselineInputsAreStable({
|
|
2651
|
-
baseModelRevision
|
|
3021
|
+
baseModelRevision,
|
|
2652
3022
|
sourceFingerprint: args.sourceFingerprint,
|
|
2653
3023
|
config: args.config
|
|
2654
3024
|
});
|
|
2655
3025
|
const cacheKey = cacheEligible ? baselineCacheKey({
|
|
2656
3026
|
modelId: args.modelId,
|
|
2657
|
-
baseModelRevision
|
|
3027
|
+
baseModelRevision,
|
|
2658
3028
|
sourceFingerprint: args.sourceFingerprint,
|
|
2659
3029
|
system: args.system,
|
|
2660
3030
|
examples,
|
|
@@ -2685,8 +3055,9 @@ async function evaluateExamples(args) {
|
|
|
2685
3055
|
const inference = args.config.dryRun ? null : await runTransformersInference({
|
|
2686
3056
|
kind: args.kind,
|
|
2687
3057
|
modelId: args.modelId,
|
|
2688
|
-
baseModelId
|
|
2689
|
-
|
|
3058
|
+
baseModelId,
|
|
3059
|
+
modelSource: args.modelSource ?? args.modelId,
|
|
3060
|
+
baseModelRevision,
|
|
2690
3061
|
adapterPath: args.adapterPath,
|
|
2691
3062
|
examples,
|
|
2692
3063
|
system: args.system,
|
|
@@ -2761,6 +3132,7 @@ function buildTrainingHyperparameters(request, options = {}) {
|
|
|
2761
3132
|
const hyper = request.hyperparameters;
|
|
2762
3133
|
return {
|
|
2763
3134
|
base_model: model.id,
|
|
3135
|
+
model_loader: model.modelLoader,
|
|
2764
3136
|
...options.baseModelRevision ? { base_model_revision: options.baseModelRevision } : {},
|
|
2765
3137
|
n_epochs: String(hyper.n_epochs),
|
|
2766
3138
|
learning_rate: String(hyper.learning_rate ?? model.defaultLearningRate),
|
|
@@ -2771,7 +3143,9 @@ function buildTrainingHyperparameters(request, options = {}) {
|
|
|
2771
3143
|
lora_rank: String(hyper.lora_rank ?? model.defaultLoraRank),
|
|
2772
3144
|
lora_alpha: String(hyper.lora_alpha ?? model.defaultLoraAlpha),
|
|
2773
3145
|
lora_dropout: String(hyper.lora_dropout ?? model.defaultLoraDropout),
|
|
2774
|
-
max_seq_length: String(hyper.max_seq_length ?? model.defaultMaxSeqLength)
|
|
3146
|
+
max_seq_length: String(hyper.max_seq_length ?? model.defaultMaxSeqLength),
|
|
3147
|
+
lora_target_modules: Array.isArray(model.loraTargetModules) ? model.loraTargetModules.join(",") : model.loraTargetModules,
|
|
3148
|
+
gradient_checkpointing: String(model.gradientCheckpointing)
|
|
2775
3149
|
};
|
|
2776
3150
|
}
|
|
2777
3151
|
async function pathExists(path) {
|
|
@@ -2884,6 +3258,13 @@ async function launchProcessTraining(args) {
|
|
|
2884
3258
|
if (value === void 0) delete childEnv[key];
|
|
2885
3259
|
}
|
|
2886
3260
|
if (config.paths.modelCache) await mkdir7(resolve10(config.paths.modelCache), { recursive: true });
|
|
3261
|
+
if (config.paths.baseModel) {
|
|
3262
|
+
await verifyLocalBaseModel(
|
|
3263
|
+
resolve10(config.paths.baseModel),
|
|
3264
|
+
request.spec_snapshot.base_model,
|
|
3265
|
+
config.paths.modelCache
|
|
3266
|
+
);
|
|
3267
|
+
}
|
|
2887
3268
|
await args.reporter?.onEvent?.({
|
|
2888
3269
|
stage: "training",
|
|
2889
3270
|
status: "running",
|
|
@@ -3206,7 +3587,7 @@ function canonicalJson2(value) {
|
|
|
3206
3587
|
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(object[key])}`).join(",")}}`;
|
|
3207
3588
|
}
|
|
3208
3589
|
function hashJson(value) {
|
|
3209
|
-
return
|
|
3590
|
+
return createHash6("sha256").update(canonicalJson2(value)).digest("hex");
|
|
3210
3591
|
}
|
|
3211
3592
|
var packageRoot2 = localRuntimePackageRoot(import.meta.url);
|
|
3212
3593
|
async function hashFileIfPresent(path) {
|
|
@@ -3246,8 +3627,8 @@ function preparedSourceFingerprint(args) {
|
|
|
3246
3627
|
});
|
|
3247
3628
|
}
|
|
3248
3629
|
async function hashFile(path) {
|
|
3249
|
-
const hash =
|
|
3250
|
-
for await (const chunk of
|
|
3630
|
+
const hash = createHash6("sha256");
|
|
3631
|
+
for await (const chunk of createReadStream3(path)) hash.update(chunk);
|
|
3251
3632
|
return hash.digest("hex");
|
|
3252
3633
|
}
|
|
3253
3634
|
async function datasetFingerprints(request) {
|
|
@@ -3266,24 +3647,28 @@ async function datasetFingerprints(request) {
|
|
|
3266
3647
|
return fingerprints;
|
|
3267
3648
|
}
|
|
3268
3649
|
async function resolveBaseModelRevision(request, config) {
|
|
3269
|
-
const
|
|
3270
|
-
|
|
3650
|
+
const requested = resolveRequestedBaseModelRevision(
|
|
3651
|
+
request.spec_snapshot.base_model,
|
|
3652
|
+
request.hyperparameters.base_model_revision
|
|
3653
|
+
);
|
|
3271
3654
|
if (config.paths.baseModel) {
|
|
3272
|
-
|
|
3273
|
-
|
|
3655
|
+
return defaultBaseModelRevision(request.spec_snapshot.base_model);
|
|
3656
|
+
}
|
|
3657
|
+
if (requested) return requested;
|
|
3658
|
+
if (!request.spec_snapshot.base_model.includes("/")) {
|
|
3659
|
+
return defaultBaseModelRevision(request.spec_snapshot.base_model);
|
|
3274
3660
|
}
|
|
3275
|
-
if (!request.spec_snapshot.base_model.includes("/")) return void 0;
|
|
3276
3661
|
const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
|
|
3277
3662
|
const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
|
|
3278
3663
|
const refPath = resolve11(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
|
|
3279
3664
|
try {
|
|
3280
3665
|
const revision = (await readFile8(refPath, "utf8")).trim();
|
|
3281
|
-
|
|
3666
|
+
if (revision) return revision;
|
|
3282
3667
|
} catch {
|
|
3283
|
-
return void 0;
|
|
3284
3668
|
}
|
|
3669
|
+
return defaultBaseModelRevision(request.spec_snapshot.base_model);
|
|
3285
3670
|
}
|
|
3286
|
-
async function fingerprintLocalBaseModel(uri) {
|
|
3671
|
+
async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
|
|
3287
3672
|
const root = localModelArtifactPath(uri);
|
|
3288
3673
|
const rootMetadata = await lstat4(root);
|
|
3289
3674
|
if (rootMetadata.isSymbolicLink()) {
|
|
@@ -3292,7 +3677,7 @@ async function fingerprintLocalBaseModel(uri) {
|
|
|
3292
3677
|
if (!rootMetadata.isDirectory()) {
|
|
3293
3678
|
throw new Error(`Local base model must be a Hugging Face snapshot directory: ${root}`);
|
|
3294
3679
|
}
|
|
3295
|
-
await verifyLocalBaseModel(root);
|
|
3680
|
+
await verifyLocalBaseModel(root, expectedModelId, modelCache);
|
|
3296
3681
|
const files = [];
|
|
3297
3682
|
const visit = async (directory) => {
|
|
3298
3683
|
for (const entry of await readdir5(directory, { withFileTypes: true })) {
|
|
@@ -3307,7 +3692,7 @@ async function fingerprintLocalBaseModel(uri) {
|
|
|
3307
3692
|
}
|
|
3308
3693
|
if (!childMetadata.isFile()) continue;
|
|
3309
3694
|
files.push({
|
|
3310
|
-
path:
|
|
3695
|
+
path: relative3(root, child).split("\\").join("/"),
|
|
3311
3696
|
size_bytes: childMetadata.size,
|
|
3312
3697
|
sha256: await hashFile(child)
|
|
3313
3698
|
});
|
|
@@ -3390,7 +3775,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
3390
3775
|
await removeReport();
|
|
3391
3776
|
}
|
|
3392
3777
|
function manifestRelativePath(artifacts, path) {
|
|
3393
|
-
return
|
|
3778
|
+
return relative3(artifacts.runDir, path).split("\\").join("/");
|
|
3394
3779
|
}
|
|
3395
3780
|
async function verifyReusableArtifacts(artifacts, paths, options = {}) {
|
|
3396
3781
|
if (!await pathExists2(artifacts.artifactManifestJson)) return false;
|
|
@@ -3487,7 +3872,7 @@ async function modelManifestContract(prepared, training) {
|
|
|
3487
3872
|
format,
|
|
3488
3873
|
framework: "transformers-peft",
|
|
3489
3874
|
base_model: prepared.request.spec_snapshot.base_model,
|
|
3490
|
-
base_model_revision: prepared.metadata.base_model_revision ??
|
|
3875
|
+
base_model_revision: prepared.metadata.base_model_revision ?? void 0,
|
|
3491
3876
|
base_model_artifact_uri: training.base_model_artifact_uri,
|
|
3492
3877
|
base_model_fingerprint: prepared.metadata.base_model_fingerprint ?? void 0,
|
|
3493
3878
|
artifact_uri: training.model_artifact_uri,
|
|
@@ -3673,7 +4058,11 @@ async function computePreparedRun(args) {
|
|
|
3673
4058
|
const requestFingerprint = hashJson(request);
|
|
3674
4059
|
const runtimeFingerprintValue = await runtimeFingerprint();
|
|
3675
4060
|
const baseModelRevision = await resolveBaseModelRevision(request, config);
|
|
3676
|
-
const baseModelFingerprint = config.paths.baseModel ? await fingerprintLocalBaseModel(
|
|
4061
|
+
const baseModelFingerprint = config.paths.baseModel ? await fingerprintLocalBaseModel(
|
|
4062
|
+
config.paths.baseModel,
|
|
4063
|
+
request.spec_snapshot.base_model,
|
|
4064
|
+
config.paths.modelCache
|
|
4065
|
+
) : void 0;
|
|
3677
4066
|
const metadata = {
|
|
3678
4067
|
run_id: request.run_id,
|
|
3679
4068
|
behavior_spec_id: request.behavior_spec_id,
|
|
@@ -3706,7 +4095,7 @@ async function computePreparedRun(args) {
|
|
|
3706
4095
|
base_model_for_evaluation: baseModelForEvaluation,
|
|
3707
4096
|
base_model_revision: baseModelRevision ?? null,
|
|
3708
4097
|
base_model_fingerprint: baseModelFingerprint ?? null,
|
|
3709
|
-
system_prompt_sha256:
|
|
4098
|
+
system_prompt_sha256: createHash6("sha256").update(system).digest("hex"),
|
|
3710
4099
|
prepared_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
3711
4100
|
};
|
|
3712
4101
|
if (args.writeArtifacts) await writeJsonAtomic(artifacts.stageMetadataJson, metadata);
|
|
@@ -3810,7 +4199,8 @@ async function runBaselineStage(args) {
|
|
|
3810
4199
|
const report = await evaluateExamples({
|
|
3811
4200
|
kind: "baseline",
|
|
3812
4201
|
modelId: args.prepared.baseModelForEvaluation,
|
|
3813
|
-
baseModelId: args.prepared.
|
|
4202
|
+
baseModelId: args.prepared.request.spec_snapshot.base_model,
|
|
4203
|
+
modelSource: args.prepared.baseModelForEvaluation,
|
|
3814
4204
|
baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
|
|
3815
4205
|
sourceFingerprint: args.prepared.metadata.base_model_fingerprint ?? void 0,
|
|
3816
4206
|
examples: args.prepared.examples,
|
|
@@ -3839,7 +4229,8 @@ async function runBaselineStage(args) {
|
|
|
3839
4229
|
await evaluateExamples({
|
|
3840
4230
|
kind: "baseline",
|
|
3841
4231
|
modelId: args.prepared.baseModelForEvaluation,
|
|
3842
|
-
baseModelId: args.prepared.
|
|
4232
|
+
baseModelId: args.prepared.request.spec_snapshot.base_model,
|
|
4233
|
+
modelSource: args.prepared.baseModelForEvaluation,
|
|
3843
4234
|
baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
|
|
3844
4235
|
sourceFingerprint: args.prepared.metadata.base_model_fingerprint ?? void 0,
|
|
3845
4236
|
examples: args.prepared.generalRegression.examples,
|
|
@@ -3901,7 +4292,9 @@ async function runTrainStage(args) {
|
|
|
3901
4292
|
request: args.prepared.request,
|
|
3902
4293
|
artifacts: args.prepared.artifacts,
|
|
3903
4294
|
config: args.config,
|
|
3904
|
-
|
|
4295
|
+
// Python loads a configured snapshot by local path, but still needs this
|
|
4296
|
+
// certified revision to enforce registry-pinned model provenance.
|
|
4297
|
+
baseModelRevision: args.prepared.metadata.base_model_revision ?? void 0,
|
|
3905
4298
|
reporter: args.runReporter,
|
|
3906
4299
|
shouldCancel: () => args.store.isCancellationRequested(args.prepared.request.run_id)
|
|
3907
4300
|
});
|
|
@@ -4005,7 +4398,8 @@ async function runCandidateStage(args) {
|
|
|
4005
4398
|
const report = await evaluateExamples({
|
|
4006
4399
|
kind: "candidate",
|
|
4007
4400
|
modelId: modelArtifact,
|
|
4008
|
-
baseModelId: args.prepared.
|
|
4401
|
+
baseModelId: args.prepared.request.spec_snapshot.base_model,
|
|
4402
|
+
modelSource: args.prepared.baseModelForEvaluation,
|
|
4009
4403
|
baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
|
|
4010
4404
|
adapterPath: modelArtifact,
|
|
4011
4405
|
examples: args.prepared.examples,
|
|
@@ -4034,7 +4428,8 @@ async function runCandidateStage(args) {
|
|
|
4034
4428
|
await evaluateExamples({
|
|
4035
4429
|
kind: "candidate",
|
|
4036
4430
|
modelId: modelArtifact,
|
|
4037
|
-
baseModelId: args.prepared.
|
|
4431
|
+
baseModelId: args.prepared.request.spec_snapshot.base_model,
|
|
4432
|
+
modelSource: args.prepared.baseModelForEvaluation,
|
|
4038
4433
|
baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
|
|
4039
4434
|
adapterPath: modelArtifact,
|
|
4040
4435
|
examples: args.prepared.generalRegression.examples,
|
|
@@ -4336,7 +4731,11 @@ async function runLocalFineTune(input) {
|
|
|
4336
4731
|
export {
|
|
4337
4732
|
writeJsonAtomic,
|
|
4338
4733
|
assertArtifactManifest,
|
|
4734
|
+
TRAINING_MODELS,
|
|
4339
4735
|
resolveTrainingModel,
|
|
4736
|
+
resolveRequestedBaseModelRevision,
|
|
4737
|
+
defaultBaseModelRevision,
|
|
4738
|
+
resolveModelLoader,
|
|
4340
4739
|
assertUsableModelArtifact,
|
|
4341
4740
|
datasetFormatSchema,
|
|
4342
4741
|
baseModelRevisionSchema,
|
|
@@ -4383,6 +4782,7 @@ export {
|
|
|
4383
4782
|
withHuggingFaceCacheEnvironment,
|
|
4384
4783
|
withOfflineHuggingFaceCacheEnvironment,
|
|
4385
4784
|
tokenF1,
|
|
4785
|
+
verifyModelPrefetchCacheReport,
|
|
4386
4786
|
buildModelPrefetchPayload,
|
|
4387
4787
|
verifyLocalBaseModel,
|
|
4388
4788
|
prefetchBaseModel,
|
|
@@ -4397,4 +4797,4 @@ export {
|
|
|
4397
4797
|
runLocalPipeline,
|
|
4398
4798
|
runLocalFineTune
|
|
4399
4799
|
};
|
|
4400
|
-
//# sourceMappingURL=chunk-
|
|
4800
|
+
//# sourceMappingURL=chunk-HDGBDHOB.js.map
|