@tuned-tensor/cli 0.13.1-beta.4 → 0.13.1-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -8
- package/dist/{chunk-7VIUHQBI.js → chunk-W7MOBPCO.js} +1544 -814
- package/dist/chunk-W7MOBPCO.js.map +1 -0
- package/dist/index.js +275 -33
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +99 -6
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/architecture.md +10 -6
- package/docs/local-runtime/foundation-long-runs.md +108 -0
- package/docs/local-runtime/spark.md +6 -0
- package/package.json +1 -1
- package/training/foundation/src/checkpoint.py +235 -0
- package/training/foundation/src/common.py +30 -2
- package/training/foundation/src/data.py +89 -3
- package/training/foundation/src/evaluate.py +10 -2
- package/training/foundation/src/pretrain.py +426 -27
- package/training/foundation/src/train_tokenizer.py +29 -5
- package/dist/chunk-7VIUHQBI.js.map +0 -1
|
@@ -477,37 +477,37 @@ async function verifyTarGzipArchive(path) {
|
|
|
477
477
|
if (sizeText && !/^[0-7]+$/.test(sizeText)) throw new Error(`Invalid tar member size in ${path}.`);
|
|
478
478
|
const size = sizeText ? Number.parseInt(sizeText, 8) : 0;
|
|
479
479
|
if (!Number.isSafeInteger(size) || size < 0) throw new Error(`Invalid tar member size in ${path}.`);
|
|
480
|
-
const
|
|
481
|
-
if (
|
|
482
|
-
throw new Error(`Unsafe or unsupported tar member type ${JSON.stringify(
|
|
480
|
+
const type2 = String.fromCharCode(header[156] ?? 0);
|
|
481
|
+
if (type2 !== "\0" && type2 !== "0" && type2 !== "5" && type2 !== "x" && type2 !== "g") {
|
|
482
|
+
throw new Error(`Unsafe or unsupported tar member type ${JSON.stringify(type2)} in ${path}.`);
|
|
483
483
|
}
|
|
484
484
|
const shortName = header.subarray(0, 100).toString("utf8").replace(/\0.*$/, "");
|
|
485
485
|
const prefix = header.subarray(345, 500).toString("utf8").replace(/\0.*$/, "");
|
|
486
486
|
const headerPath = safeMemberPath(prefix ? `${prefix}/${shortName}` : shortName);
|
|
487
|
-
const memberPath =
|
|
488
|
-
if (
|
|
487
|
+
const memberPath = type2 === "x" || type2 === "g" ? headerPath : safeMemberPath(pendingPaxPath ?? headerPath);
|
|
488
|
+
if (type2 !== "x" && type2 !== "g") pendingPaxPath = void 0;
|
|
489
489
|
const segments = memberPath.split("/").filter(Boolean);
|
|
490
490
|
const name = segments.at(-1)?.toLowerCase();
|
|
491
491
|
const metadataOnly = name === "training-metrics.json" || name === "trainer_state.json" || name === "training_args.bin" || name === "config.json" || name === "generation_config.json" || name === "adapter_config.json" || name === "readme.md" || name?.startsWith("tokenizer") || name?.startsWith("special_tokens") || name?.startsWith("added_tokens") || name === "vocab.json" || name === "merges.txt";
|
|
492
492
|
const recognizedPayload = Boolean(name) && name !== "training_args.bin" && (name.endsWith(".safetensors") || name.endsWith(".bin"));
|
|
493
493
|
const adapterWeight = name === "adapter_model.safetensors" || name === "adapter_model.bin";
|
|
494
494
|
const fullModelWeight = Boolean(name) && (name.endsWith(".safetensors") && !name.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(name));
|
|
495
|
-
if (size > 0 && name === "adapter_config.json" && (
|
|
495
|
+
if (size > 0 && name === "adapter_config.json" && (type2 === "\0" || type2 === "0")) {
|
|
496
496
|
adapterConfigEntries += 1;
|
|
497
497
|
}
|
|
498
|
-
if ((
|
|
498
|
+
if ((type2 === "x" || type2 === "g") && size > maxPaxBytes) {
|
|
499
499
|
throw new Error(`PAX metadata exceeds 1 MiB in ${path}.`);
|
|
500
500
|
}
|
|
501
|
-
if (size > 0 && !metadataOnly && (
|
|
502
|
-
if (size > 0 && recognizedPayload && (
|
|
501
|
+
if (size > 0 && !metadataOnly && (type2 === "\0" || type2 === "0")) payloadEntries += 1;
|
|
502
|
+
if (size > 0 && recognizedPayload && (type2 === "\0" || type2 === "0")) {
|
|
503
503
|
recognizedPayloadEntries += 1;
|
|
504
504
|
recognizedPayloadBytes += size;
|
|
505
505
|
}
|
|
506
|
-
if (size > 0 && adapterWeight && (
|
|
506
|
+
if (size > 0 && adapterWeight && (type2 === "\0" || type2 === "0")) {
|
|
507
507
|
adapterWeightEntries += 1;
|
|
508
508
|
adapterWeightBytes += size;
|
|
509
509
|
}
|
|
510
|
-
if (size > 0 && fullModelWeight && (
|
|
510
|
+
if (size > 0 && fullModelWeight && (type2 === "\0" || type2 === "0")) {
|
|
511
511
|
fullModelWeightEntries += 1;
|
|
512
512
|
fullModelWeightBytes += size;
|
|
513
513
|
}
|
|
@@ -515,7 +515,7 @@ async function verifyTarGzipArchive(path) {
|
|
|
515
515
|
if (expandedBytes > maxExpandedBytes) throw new Error(`Tar archive exceeds 20 GiB expanded size: ${path}.`);
|
|
516
516
|
payloadBytes = size;
|
|
517
517
|
paddingBytes = (512 - size % 512) % 512;
|
|
518
|
-
paxType =
|
|
518
|
+
paxType = type2 === "x" || type2 === "g" ? type2 : null;
|
|
519
519
|
paxChunks = [];
|
|
520
520
|
entries += 1;
|
|
521
521
|
if (entries > maxEntries) throw new Error(`Tar archive exceeds ${maxEntries} members: ${path}.`);
|
|
@@ -834,28 +834,371 @@ async function assertUsableModelArtifact(uri) {
|
|
|
834
834
|
return inspection;
|
|
835
835
|
}
|
|
836
836
|
|
|
837
|
-
// src/local-runtime/
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
837
|
+
// src/local-runtime/run-reporter.ts
|
|
838
|
+
function reportInBackground(action) {
|
|
839
|
+
try {
|
|
840
|
+
void Promise.resolve(action()).catch(() => void 0);
|
|
841
|
+
} catch {
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
function sanitizeLogLine(line) {
|
|
845
|
+
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
846
|
+
}
|
|
847
|
+
function forwardStreamLines(stream, onLine) {
|
|
848
|
+
let buffered = "";
|
|
849
|
+
stream.setEncoding("utf8");
|
|
850
|
+
stream.on("data", (chunk) => {
|
|
851
|
+
buffered += chunk;
|
|
852
|
+
const lines = buffered.split(/\r\n|\n|\r/);
|
|
853
|
+
buffered = lines.pop() ?? "";
|
|
854
|
+
for (const line of lines) {
|
|
855
|
+
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
856
|
+
}
|
|
857
|
+
});
|
|
858
|
+
stream.on("end", () => {
|
|
859
|
+
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
860
|
+
});
|
|
861
|
+
}
|
|
841
862
|
|
|
842
|
-
// src/
|
|
863
|
+
// src/local-runtime/package-root.ts
|
|
843
864
|
import { existsSync } from "fs";
|
|
865
|
+
import { dirname as dirname2, resolve as resolve3 } from "path";
|
|
866
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
867
|
+
function localRuntimePackageRoot(moduleUrl) {
|
|
868
|
+
const moduleDirectory = dirname2(fileURLToPath2(moduleUrl));
|
|
869
|
+
const candidates = [
|
|
870
|
+
dirname2(moduleDirectory),
|
|
871
|
+
dirname2(dirname2(moduleDirectory))
|
|
872
|
+
];
|
|
873
|
+
const root = candidates.find(
|
|
874
|
+
(candidate) => existsSync(resolve3(candidate, "package.json")) && existsSync(resolve3(candidate, "training/adapter"))
|
|
875
|
+
);
|
|
876
|
+
if (!root) {
|
|
877
|
+
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
878
|
+
}
|
|
879
|
+
return root;
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
// src/local-runtime/capability-profiles.ts
|
|
883
|
+
var GIB = 1024 ** 3;
|
|
884
|
+
function gib(n) {
|
|
885
|
+
return Math.round(n * GIB);
|
|
886
|
+
}
|
|
887
|
+
var FOUNDATION_DEFAULT_DEPTH = 2;
|
|
888
|
+
var FOUNDATION_MAX_DEPTH = 64;
|
|
889
|
+
var FOUNDATION_DEFAULT_VOCAB = 256;
|
|
890
|
+
var FOUNDATION_DEFAULT_SEQUENCE = 64;
|
|
891
|
+
var FOUNDATION_DEFAULT_BATCH = 2;
|
|
892
|
+
var ADAPTER_MEMORY_PROFILES = [
|
|
893
|
+
{
|
|
894
|
+
id: "Qwen/Qwen3.5-2B",
|
|
895
|
+
diskBytes: gib(8),
|
|
896
|
+
inferenceBytes: gib(8),
|
|
897
|
+
trainBytes: gib(16),
|
|
898
|
+
sparkClass: false
|
|
899
|
+
},
|
|
900
|
+
{
|
|
901
|
+
id: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
|
|
902
|
+
diskBytes: gib(80),
|
|
903
|
+
inferenceBytes: gib(80),
|
|
904
|
+
trainBytes: gib(96),
|
|
905
|
+
sparkClass: true
|
|
906
|
+
},
|
|
907
|
+
{
|
|
908
|
+
id: "meta-models/Muse-Glimmer-30B",
|
|
909
|
+
diskBytes: gib(80),
|
|
910
|
+
inferenceBytes: gib(80),
|
|
911
|
+
trainBytes: gib(96),
|
|
912
|
+
sparkClass: true
|
|
913
|
+
}
|
|
914
|
+
];
|
|
915
|
+
function adapterMemoryProfile(modelId) {
|
|
916
|
+
const normalized = modelId.trim().toLowerCase();
|
|
917
|
+
return ADAPTER_MEMORY_PROFILES.find((profile) => profile.id.toLowerCase() === normalized);
|
|
918
|
+
}
|
|
919
|
+
function foundationWidth(depth) {
|
|
920
|
+
return 32 * depth;
|
|
921
|
+
}
|
|
922
|
+
function foundationParamCount(depth, vocabSize = FOUNDATION_DEFAULT_VOCAB, sequenceLength = FOUNDATION_DEFAULT_SEQUENCE) {
|
|
923
|
+
const width = foundationWidth(depth);
|
|
924
|
+
return 12 * depth * width * width + vocabSize * width + sequenceLength * width;
|
|
925
|
+
}
|
|
926
|
+
function foundationTrainBytes(depth, options = {}) {
|
|
927
|
+
const vocabSize = options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB;
|
|
928
|
+
const sequenceLength = options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE;
|
|
929
|
+
const batchSize = options.batchSize ?? FOUNDATION_DEFAULT_BATCH;
|
|
930
|
+
const params = foundationParamCount(depth, vocabSize, sequenceLength);
|
|
931
|
+
const width = foundationWidth(depth);
|
|
932
|
+
const activationSlack = batchSize * sequenceLength * width * depth * 64;
|
|
933
|
+
return 16 * params + activationSlack;
|
|
934
|
+
}
|
|
935
|
+
function foundationInferenceBytes(depth, options = {}) {
|
|
936
|
+
const params = foundationParamCount(
|
|
937
|
+
depth,
|
|
938
|
+
options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB,
|
|
939
|
+
options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE
|
|
940
|
+
);
|
|
941
|
+
return 4 * params + gib(0.25);
|
|
942
|
+
}
|
|
943
|
+
function isUnifiedGpuName(name) {
|
|
944
|
+
if (!name) return false;
|
|
945
|
+
return /spark|gb10/i.test(name);
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
// src/local-runtime/capability.ts
|
|
949
|
+
var HEADROOM = 1.2;
|
|
950
|
+
var FIT_RATIO = 0.7;
|
|
951
|
+
function formatGiB(bytes) {
|
|
952
|
+
const amount = bytes / 1024 ** 3;
|
|
953
|
+
const precision = amount >= 10 ? 0 : 1;
|
|
954
|
+
return `${amount.toFixed(precision)} GiB`;
|
|
955
|
+
}
|
|
956
|
+
function compareFit(available, required, missing) {
|
|
957
|
+
if (available === void 0) {
|
|
958
|
+
return { status: "not_possible", reason: missing, required_bytes: required };
|
|
959
|
+
}
|
|
960
|
+
if (available < required) {
|
|
961
|
+
return {
|
|
962
|
+
status: "not_possible",
|
|
963
|
+
reason: `needs ~${formatGiB(required)}; ${formatGiB(available)} available`,
|
|
964
|
+
required_bytes: required,
|
|
965
|
+
available_bytes: available
|
|
966
|
+
};
|
|
967
|
+
}
|
|
968
|
+
if (available < required * HEADROOM) {
|
|
969
|
+
return {
|
|
970
|
+
status: "tight",
|
|
971
|
+
reason: `fits in ${formatGiB(available)} with little headroom (needs ~${formatGiB(required)})`,
|
|
972
|
+
required_bytes: required,
|
|
973
|
+
available_bytes: available
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
return {
|
|
977
|
+
status: "ready",
|
|
978
|
+
reason: `${formatGiB(available)} available (needs ~${formatGiB(required)})`,
|
|
979
|
+
required_bytes: required,
|
|
980
|
+
available_bytes: available
|
|
981
|
+
};
|
|
982
|
+
}
|
|
983
|
+
function trainingBlockers(inventory) {
|
|
984
|
+
const blockers = [];
|
|
985
|
+
if (!inventory.node.ok) blockers.push(`Node ${inventory.node.version} is below 22`);
|
|
986
|
+
if (inventory.uv && !inventory.uv.ok) blockers.push(inventory.uv.message);
|
|
987
|
+
if (inventory.python && !inventory.python.ok) blockers.push(inventory.python.message);
|
|
988
|
+
return blockers;
|
|
989
|
+
}
|
|
990
|
+
function cudaAvailable(inventory) {
|
|
991
|
+
if (inventory.python?.ok && inventory.python.cuda_available !== void 0) {
|
|
992
|
+
return inventory.python.cuda_available;
|
|
993
|
+
}
|
|
994
|
+
return inventory.nvidia_smi.ok && inventory.gpus.length > 0;
|
|
995
|
+
}
|
|
996
|
+
function trainingGpu(inventory) {
|
|
997
|
+
const listed = inventory.gpus.find((gpu) => gpu.index === 0) ?? inventory.gpus[0];
|
|
998
|
+
if (listed) {
|
|
999
|
+
if (!listed.memory_total_bytes && inventory.python?.total_memory_bytes) {
|
|
1000
|
+
return { ...listed, memory_total_bytes: inventory.python.total_memory_bytes };
|
|
1001
|
+
}
|
|
1002
|
+
return listed;
|
|
1003
|
+
}
|
|
1004
|
+
if (inventory.python?.cuda_device) {
|
|
1005
|
+
return {
|
|
1006
|
+
index: 0,
|
|
1007
|
+
name: inventory.python.cuda_device,
|
|
1008
|
+
memory_total_bytes: inventory.python.total_memory_bytes,
|
|
1009
|
+
unified_memory: isUnifiedGpuName(inventory.python.cuda_device)
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
return void 0;
|
|
1013
|
+
}
|
|
1014
|
+
function gpuMemory(gpu) {
|
|
1015
|
+
return gpu?.memory_total_bytes;
|
|
1016
|
+
}
|
|
1017
|
+
function diskForCache(inventory) {
|
|
1018
|
+
return inventory.disks.find((disk) => disk.name === "model-cache")?.free_bytes;
|
|
1019
|
+
}
|
|
1020
|
+
function worse(left, right) {
|
|
1021
|
+
const rank = { not_possible: 2, tight: 1, ready: 0 };
|
|
1022
|
+
return rank[left.status] >= rank[right.status] ? left : right;
|
|
1023
|
+
}
|
|
1024
|
+
function withBlockers(verdict, blockers) {
|
|
1025
|
+
if (blockers.length === 0) return verdict;
|
|
1026
|
+
return {
|
|
1027
|
+
status: "not_possible",
|
|
1028
|
+
reason: blockers.join("; "),
|
|
1029
|
+
required_bytes: verdict.required_bytes,
|
|
1030
|
+
available_bytes: verdict.available_bytes
|
|
1031
|
+
};
|
|
1032
|
+
}
|
|
1033
|
+
function adapterVerdicts(profile, inventory, gpu, hasCuda) {
|
|
1034
|
+
const blockers = trainingBlockers(inventory);
|
|
1035
|
+
const gpuBytes = gpuMemory(gpu);
|
|
1036
|
+
const ram = inventory.os.total_memory_bytes;
|
|
1037
|
+
const disk = diskForCache(inventory);
|
|
1038
|
+
let train;
|
|
1039
|
+
if (!hasCuda) {
|
|
1040
|
+
train = {
|
|
1041
|
+
status: "not_possible",
|
|
1042
|
+
reason: "LoRA training requires CUDA",
|
|
1043
|
+
required_bytes: profile.trainBytes
|
|
1044
|
+
};
|
|
1045
|
+
} else {
|
|
1046
|
+
train = compareFit(gpuBytes, profile.trainBytes, "GPU memory is unknown");
|
|
1047
|
+
}
|
|
1048
|
+
train = withBlockers(train, blockers);
|
|
1049
|
+
if (disk !== void 0 && disk < profile.diskBytes && train.status !== "not_possible") {
|
|
1050
|
+
train = worse(train, compareFit(disk, profile.diskBytes, "model cache disk is unknown"));
|
|
1051
|
+
}
|
|
1052
|
+
const finetune = train;
|
|
1053
|
+
let inference;
|
|
1054
|
+
if (hasCuda) {
|
|
1055
|
+
inference = compareFit(gpuBytes, profile.inferenceBytes, "GPU memory is unknown");
|
|
1056
|
+
} else {
|
|
1057
|
+
inference = compareFit(ram, profile.inferenceBytes, "system RAM is unknown");
|
|
1058
|
+
if (inference.status !== "not_possible") {
|
|
1059
|
+
inference = {
|
|
1060
|
+
...inference,
|
|
1061
|
+
reason: `CPU inference: ${inference.reason}`
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
return {
|
|
1066
|
+
id: profile.id,
|
|
1067
|
+
spark_class: profile.sparkClass,
|
|
1068
|
+
train,
|
|
1069
|
+
finetune,
|
|
1070
|
+
inference
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
function suggestedFoundationDepth(availableBytes) {
|
|
1074
|
+
if (availableBytes === void 0) return FOUNDATION_DEFAULT_DEPTH;
|
|
1075
|
+
const budget = availableBytes * FIT_RATIO;
|
|
1076
|
+
let suggested = FOUNDATION_DEFAULT_DEPTH;
|
|
1077
|
+
for (let depth = FOUNDATION_DEFAULT_DEPTH; depth <= FOUNDATION_MAX_DEPTH; depth += 1) {
|
|
1078
|
+
if (foundationTrainBytes(depth) <= budget) suggested = depth;
|
|
1079
|
+
else break;
|
|
1080
|
+
}
|
|
1081
|
+
return suggested;
|
|
1082
|
+
}
|
|
1083
|
+
function evaluateCapabilities(inventory) {
|
|
1084
|
+
const hasCuda = cudaAvailable(inventory);
|
|
1085
|
+
const gpu = trainingGpu(inventory);
|
|
1086
|
+
const notes = [];
|
|
1087
|
+
if (gpu?.unified_memory || isUnifiedGpuName(gpu?.name)) {
|
|
1088
|
+
notes.push("GPU memory is a unified pool shared with system RAM (Spark-class).");
|
|
1089
|
+
}
|
|
1090
|
+
if (inventory.quick) {
|
|
1091
|
+
notes.push("Quick probe skipped the bundled torch/uv runtime.");
|
|
1092
|
+
}
|
|
1093
|
+
if (!inventory.node.ok) {
|
|
1094
|
+
notes.push(`Node ${inventory.node.version} is below the 22.x required by the local runner.`);
|
|
1095
|
+
}
|
|
1096
|
+
const adapters = ADAPTER_MEMORY_PROFILES.map(
|
|
1097
|
+
(profile) => adapterVerdicts(profile, inventory, gpu, hasCuda)
|
|
1098
|
+
);
|
|
1099
|
+
const blockers = trainingBlockers(inventory);
|
|
1100
|
+
const gpuBytes = gpuMemory(gpu);
|
|
1101
|
+
const ram = inventory.os.total_memory_bytes;
|
|
1102
|
+
const trainBudget = hasCuda ? gpuBytes : void 0;
|
|
1103
|
+
const suggested = suggestedFoundationDepth(trainBudget);
|
|
1104
|
+
const defaultTrain = foundationTrainBytes(FOUNDATION_DEFAULT_DEPTH);
|
|
1105
|
+
const inferBudget = hasCuda ? gpuBytes : ram;
|
|
1106
|
+
let train;
|
|
1107
|
+
if (!hasCuda) {
|
|
1108
|
+
train = {
|
|
1109
|
+
status: "not_possible",
|
|
1110
|
+
reason: "Foundation pretrain, SFT, and RL require CUDA",
|
|
1111
|
+
required_bytes: defaultTrain
|
|
1112
|
+
};
|
|
1113
|
+
} else {
|
|
1114
|
+
train = compareFit(gpuBytes, defaultTrain, "GPU memory is unknown");
|
|
1115
|
+
}
|
|
1116
|
+
train = withBlockers(train, blockers);
|
|
1117
|
+
const inference = compareFit(
|
|
1118
|
+
inferBudget,
|
|
1119
|
+
foundationInferenceBytes(FOUNDATION_DEFAULT_DEPTH),
|
|
1120
|
+
hasCuda ? "GPU memory is unknown" : "system RAM is unknown"
|
|
1121
|
+
);
|
|
1122
|
+
const foundation = {
|
|
1123
|
+
default_depth: FOUNDATION_DEFAULT_DEPTH,
|
|
1124
|
+
suggested_max_depth: hasCuda && blockers.length === 0 ? suggested : FOUNDATION_DEFAULT_DEPTH,
|
|
1125
|
+
train,
|
|
1126
|
+
finetune: train,
|
|
1127
|
+
inference: hasCuda ? inference : { ...inference, reason: `CPU eval: ${inference.reason}` },
|
|
1128
|
+
serve: {
|
|
1129
|
+
status: "not_possible",
|
|
1130
|
+
reason: "tt serve cannot host foundation checkpoints yet"
|
|
1131
|
+
}
|
|
1132
|
+
};
|
|
1133
|
+
return { cuda_available: hasCuda, gpu, adapters, foundation, notes };
|
|
1134
|
+
}
|
|
1135
|
+
function formatCapabilitySummary(report) {
|
|
1136
|
+
const gpu = report.gpu;
|
|
1137
|
+
const gpuLabel = gpu ? `${gpu.name}${gpu.memory_total_bytes ? `, ${formatGiB(gpu.memory_total_bytes)}` : ""}${gpu.unified_memory ? " unified" : ""}` : "no NVIDIA GPU";
|
|
1138
|
+
const adapterBits = report.adapters.map((adapter) => {
|
|
1139
|
+
const short = adapter.id.split("/").at(-1) ?? adapter.id;
|
|
1140
|
+
return `${short} train ${adapter.train.status}`;
|
|
1141
|
+
});
|
|
1142
|
+
return [
|
|
1143
|
+
`GPU ${gpuLabel}, CUDA ${report.cuda_available ? "yes" : "no"}`,
|
|
1144
|
+
`Adapter: ${adapterBits.join("; ")}`,
|
|
1145
|
+
`Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve not supported`
|
|
1146
|
+
].join(". ");
|
|
1147
|
+
}
|
|
1148
|
+
function warningsFromSnapshot(report, target) {
|
|
1149
|
+
const warnings = [];
|
|
1150
|
+
if (target.engine === "foundation") {
|
|
1151
|
+
for (const kind of ["train", "finetune"]) {
|
|
1152
|
+
const verdict = report.foundation[kind];
|
|
1153
|
+
if (verdict.status === "not_possible" || verdict.status === "tight") {
|
|
1154
|
+
warnings.push(`Foundation ${kind} is ${verdict.status}: ${verdict.reason}`);
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
return warnings;
|
|
1158
|
+
}
|
|
1159
|
+
const modelId = target.baseModel;
|
|
1160
|
+
const adapter = modelId ? report.adapters.find((item) => item.id.toLowerCase() === modelId.trim().toLowerCase()) : void 0;
|
|
1161
|
+
if (!adapter) {
|
|
1162
|
+
if (modelId && !adapterMemoryProfile(modelId)) return warnings;
|
|
1163
|
+
const blocked = report.adapters.filter((item) => item.train.status === "not_possible");
|
|
1164
|
+
if (blocked.length === report.adapters.length) {
|
|
1165
|
+
warnings.push(`Adapter train is not_possible: ${report.adapters[0]?.train.reason ?? "CUDA required"}`);
|
|
1166
|
+
}
|
|
1167
|
+
return warnings;
|
|
1168
|
+
}
|
|
1169
|
+
for (const kind of ["train", "finetune", "inference"]) {
|
|
1170
|
+
const verdict = adapter[kind];
|
|
1171
|
+
if (verdict.status === "not_possible" || verdict.status === "tight") {
|
|
1172
|
+
warnings.push(`${adapter.id} ${kind} is ${verdict.status}: ${verdict.reason}`);
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
return warnings;
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
// src/local-runtime/host-inventory.ts
|
|
1179
|
+
import { constants as constants2 } from "fs";
|
|
1180
|
+
import { access, statfs } from "fs/promises";
|
|
1181
|
+
import { cpus, freemem, homedir as homedir3, totalmem, type } from "os";
|
|
1182
|
+
import { dirname as dirname5, join as join5, resolve as resolve7 } from "path";
|
|
1183
|
+
import { spawn as spawn2 } from "child_process";
|
|
1184
|
+
|
|
1185
|
+
// src/paths.ts
|
|
1186
|
+
import { existsSync as existsSync2 } from "fs";
|
|
844
1187
|
import { homedir } from "os";
|
|
845
|
-
import { isAbsolute as isAbsolute2, join as join2, resolve as
|
|
1188
|
+
import { isAbsolute as isAbsolute2, join as join2, resolve as resolve4 } from "path";
|
|
846
1189
|
var TUNED_TENSOR_DIR = ".tuned-tensor";
|
|
847
1190
|
var DEFAULT_ARTIFACT_ROOT = `${TUNED_TENSOR_DIR}/artifacts`;
|
|
848
1191
|
var DEFAULT_PROJECT_STORE_ROOT = `${TUNED_TENSOR_DIR}/store`;
|
|
849
1192
|
var DEFAULT_FOUNDATION_RUNS_DIR = `${TUNED_TENSOR_DIR}/foundation-runs`;
|
|
850
1193
|
function envHome(env = process.env) {
|
|
851
1194
|
const home = env.HOME?.trim();
|
|
852
|
-
return home ?
|
|
1195
|
+
return home ? resolve4(home) : homedir();
|
|
853
1196
|
}
|
|
854
1197
|
function expandUserPath(value, env = process.env, baseDirectory = process.cwd()) {
|
|
855
1198
|
const trimmed = value.trim();
|
|
856
1199
|
if (trimmed === "~") return envHome(env);
|
|
857
|
-
if (trimmed.startsWith("~/")) return
|
|
858
|
-
return isAbsolute2(trimmed) ?
|
|
1200
|
+
if (trimmed.startsWith("~/")) return resolve4(envHome(env), trimmed.slice(2));
|
|
1201
|
+
return isAbsolute2(trimmed) ? resolve4(trimmed) : resolve4(baseDirectory, trimmed);
|
|
859
1202
|
}
|
|
860
1203
|
function getTunedTensorHome(env = process.env) {
|
|
861
1204
|
const override = env.TUNED_TENSOR_HOME?.trim();
|
|
@@ -863,14 +1206,14 @@ function getTunedTensorHome(env = process.env) {
|
|
|
863
1206
|
return join2(envHome(env), TUNED_TENSOR_DIR);
|
|
864
1207
|
}
|
|
865
1208
|
function legacyConfigDir(env) {
|
|
866
|
-
const configHome = env.XDG_CONFIG_HOME?.trim() ?
|
|
1209
|
+
const configHome = env.XDG_CONFIG_HOME?.trim() ? resolve4(env.XDG_CONFIG_HOME) : join2(envHome(env), ".config");
|
|
867
1210
|
return join2(configHome, "tuned-tensor");
|
|
868
1211
|
}
|
|
869
1212
|
function getConfigDir(env = process.env) {
|
|
870
1213
|
const next = getTunedTensorHome(env);
|
|
871
1214
|
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
872
1215
|
const legacy = legacyConfigDir(env);
|
|
873
|
-
if (!
|
|
1216
|
+
if (!existsSync2(join2(next, "config.json")) && existsSync2(join2(legacy, "config.json"))) {
|
|
874
1217
|
return legacy;
|
|
875
1218
|
}
|
|
876
1219
|
return next;
|
|
@@ -879,7 +1222,7 @@ function getAgentConfigDir(env = process.env) {
|
|
|
879
1222
|
const next = join2(getTunedTensorHome(env), "agent");
|
|
880
1223
|
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
881
1224
|
const legacy = join2(legacyConfigDir(env), "agent");
|
|
882
|
-
if (!
|
|
1225
|
+
if (!existsSync2(next) && existsSync2(legacy)) return legacy;
|
|
883
1226
|
return next;
|
|
884
1227
|
}
|
|
885
1228
|
function defaultStoreRoot(env = process.env) {
|
|
@@ -888,7 +1231,7 @@ function defaultStoreRoot(env = process.env) {
|
|
|
888
1231
|
const next = join2(getTunedTensorHome(env), "store");
|
|
889
1232
|
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
890
1233
|
const legacy = join2(envHome(env), ".tuned-tensor-local");
|
|
891
|
-
if (!
|
|
1234
|
+
if (!existsSync2(next) && existsSync2(legacy)) return legacy;
|
|
892
1235
|
return next;
|
|
893
1236
|
}
|
|
894
1237
|
function defaultCacheRoot(env = process.env) {
|
|
@@ -897,179 +1240,878 @@ function defaultCacheRoot(env = process.env) {
|
|
|
897
1240
|
function pythonEnvironmentPath(kind, hash, env = process.env) {
|
|
898
1241
|
const next = join2(defaultCacheRoot(env), kind, hash);
|
|
899
1242
|
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
900
|
-
const cacheHome = env.XDG_CACHE_HOME?.trim() ?
|
|
1243
|
+
const cacheHome = env.XDG_CACHE_HOME?.trim() ? resolve4(env.XDG_CACHE_HOME) : join2(envHome(env), ".cache");
|
|
901
1244
|
const legacy = join2(cacheHome, "tuned-tensor-local", kind, hash);
|
|
902
|
-
if (!
|
|
1245
|
+
if (!existsSync2(next) && existsSync2(legacy)) return legacy;
|
|
903
1246
|
return next;
|
|
904
1247
|
}
|
|
905
1248
|
|
|
906
|
-
// src/local-runtime/
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
1249
|
+
// src/local-runtime/huggingface-cache.ts
|
|
1250
|
+
import { homedir as homedir2 } from "os";
|
|
1251
|
+
import { dirname as dirname3, join as join3, resolve as resolve5 } from "path";
|
|
1252
|
+
function minimalMachineLearningEnvironment(env) {
|
|
1253
|
+
const exact = /* @__PURE__ */ new Set([
|
|
1254
|
+
"PATH",
|
|
1255
|
+
"HOME",
|
|
1256
|
+
"USER",
|
|
1257
|
+
"LOGNAME",
|
|
1258
|
+
"SHELL",
|
|
1259
|
+
"TMPDIR",
|
|
1260
|
+
"TEMP",
|
|
1261
|
+
"TMP",
|
|
1262
|
+
"LANG",
|
|
1263
|
+
"LANGUAGE",
|
|
1264
|
+
"LD_LIBRARY_PATH",
|
|
1265
|
+
"DYLD_LIBRARY_PATH",
|
|
1266
|
+
"XDG_CACHE_HOME",
|
|
1267
|
+
"SSL_CERT_FILE",
|
|
1268
|
+
"REQUESTS_CA_BUNDLE",
|
|
1269
|
+
"CURL_CA_BUNDLE",
|
|
1270
|
+
"NO_PROXY",
|
|
1271
|
+
"no_proxy",
|
|
1272
|
+
"UV_CACHE_DIR",
|
|
1273
|
+
"UV_PYTHON",
|
|
1274
|
+
"HF_HOME",
|
|
1275
|
+
"HF_HUB_CACHE",
|
|
1276
|
+
"HUGGINGFACE_HUB_CACHE",
|
|
1277
|
+
"HF_ENDPOINT",
|
|
1278
|
+
"HF_TOKEN",
|
|
1279
|
+
"HF_HUB_OFFLINE",
|
|
1280
|
+
"HF_HUB_DISABLE_TELEMETRY",
|
|
1281
|
+
"TRANSFORMERS_OFFLINE",
|
|
1282
|
+
"HF_HUB_ENABLE_HF_TRANSFER",
|
|
1283
|
+
"PYTHONUTF8",
|
|
1284
|
+
"PYTHONIOENCODING"
|
|
1285
|
+
]);
|
|
1286
|
+
const prefixes = [
|
|
1287
|
+
"LC_",
|
|
1288
|
+
"CUDA_",
|
|
1289
|
+
"NVIDIA_",
|
|
1290
|
+
"NCCL_",
|
|
1291
|
+
"OMP_",
|
|
1292
|
+
"MKL_",
|
|
1293
|
+
"TORCH_",
|
|
1294
|
+
"PYTORCH_",
|
|
1295
|
+
"TOKENIZERS_",
|
|
1296
|
+
"HF_XET_"
|
|
1297
|
+
];
|
|
1298
|
+
const result = Object.fromEntries(Object.entries(env).filter(
|
|
1299
|
+
([key]) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix))
|
|
1300
|
+
));
|
|
1301
|
+
result.TORCH_DISABLE_NATIVE_JIT ??= "1";
|
|
1302
|
+
return result;
|
|
912
1303
|
}
|
|
913
|
-
function
|
|
1304
|
+
function resolveHuggingFaceCacheLayout(modelCache) {
|
|
1305
|
+
const hfHome = resolve5(modelCache);
|
|
914
1306
|
return {
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
runsDir: join3(root, "runs"),
|
|
918
|
-
modelsDir: join3(root, "models"),
|
|
919
|
-
datasetsDir: join3(root, "datasets")
|
|
1307
|
+
hfHome,
|
|
1308
|
+
hubCache: join3(hfHome, "hub")
|
|
920
1309
|
};
|
|
921
1310
|
}
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
if (!metadata.isDirectory()) {
|
|
937
|
-
throw new Error(`Expected local store directory: ${path}`);
|
|
938
|
-
}
|
|
939
|
-
await chmod(path, 448);
|
|
1311
|
+
function withHuggingFaceCacheEnvironment(env, modelCache) {
|
|
1312
|
+
const inheritedHub = env.HF_HUB_CACHE ?? env.HUGGINGFACE_HUB_CACHE;
|
|
1313
|
+
const canonicalHome = modelCache ?? env.HF_HOME ?? (inheritedHub ? dirname3(resolve5(inheritedHub)) : join3(homedir2(), ".cache", "huggingface"));
|
|
1314
|
+
const layout = resolveHuggingFaceCacheLayout(canonicalHome);
|
|
1315
|
+
const result = {
|
|
1316
|
+
...env,
|
|
1317
|
+
HF_HOME: layout.hfHome,
|
|
1318
|
+
HF_HUB_CACHE: layout.hubCache,
|
|
1319
|
+
HUGGINGFACE_HUB_CACHE: layout.hubCache
|
|
1320
|
+
};
|
|
1321
|
+
delete result.TRANSFORMERS_CACHE;
|
|
1322
|
+
delete result.PYTORCH_TRANSFORMERS_CACHE;
|
|
1323
|
+
delete result.PYTORCH_PRETRAINED_BERT_CACHE;
|
|
1324
|
+
return result;
|
|
940
1325
|
}
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
await chmod(path, 448);
|
|
948
|
-
for (const entry of await readdir3(path)) {
|
|
949
|
-
await hardenPrivateTree(join3(path, entry));
|
|
950
|
-
}
|
|
951
|
-
return;
|
|
952
|
-
}
|
|
953
|
-
if (metadata.isFile()) {
|
|
954
|
-
await chmod(path, 384);
|
|
955
|
-
return;
|
|
956
|
-
}
|
|
957
|
-
throw new Error(`Refusing unsupported filesystem entry in local store: ${path}`);
|
|
1326
|
+
function withOfflineHuggingFaceCacheEnvironment(env, modelCache) {
|
|
1327
|
+
return {
|
|
1328
|
+
...withHuggingFaceCacheEnvironment(env, modelCache),
|
|
1329
|
+
HF_HUB_OFFLINE: "1",
|
|
1330
|
+
TRANSFORMERS_OFFLINE: "1"
|
|
1331
|
+
};
|
|
958
1332
|
}
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
1333
|
+
|
|
1334
|
+
// src/local-runtime/process-runner.ts
|
|
1335
|
+
import { createHash as createHash2 } from "crypto";
|
|
1336
|
+
import { spawn } from "child_process";
|
|
1337
|
+
import { constants, readFileSync } from "fs";
|
|
1338
|
+
import { mkdir as mkdir2, open as open2 } from "fs/promises";
|
|
1339
|
+
import { dirname as dirname4, join as join4, resolve as resolve6 } from "path";
|
|
1340
|
+
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
1341
|
+
var bundledProject = join4(packageRoot, "training/adapter");
|
|
1342
|
+
var bundledRuntimeHash = (() => {
|
|
1343
|
+
const hash = createHash2("sha256");
|
|
1344
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1345
|
+
hash.update(readFileSync(join4(bundledProject, name)));
|
|
970
1346
|
}
|
|
971
|
-
|
|
972
|
-
|
|
1347
|
+
return hash.digest("hex").slice(0, 20);
|
|
1348
|
+
})();
|
|
1349
|
+
var BUNDLED_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv", bundledRuntimeHash);
|
|
1350
|
+
var foundationProject = join4(packageRoot, "training/foundation");
|
|
1351
|
+
var foundationRuntimeHash = (() => {
|
|
1352
|
+
const hash = createHash2("sha256");
|
|
1353
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1354
|
+
hash.update(readFileSync(join4(foundationProject, name)));
|
|
973
1355
|
}
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
const
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
return
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
`, {
|
|
995
|
-
encoding: "utf8",
|
|
996
|
-
mode: 384
|
|
997
|
-
});
|
|
998
|
-
await chmod(path, 384);
|
|
1356
|
+
return hash.digest("hex").slice(0, 20);
|
|
1357
|
+
})();
|
|
1358
|
+
var FOUNDATION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-foundation", foundationRuntimeHash);
|
|
1359
|
+
function buildBundledPythonCommand(entrypoint, args = []) {
|
|
1360
|
+
const target = entrypoint === "-c" ? entrypoint : join4(bundledProject, "src", entrypoint);
|
|
1361
|
+
const commandArgs = [
|
|
1362
|
+
"run",
|
|
1363
|
+
"--frozen",
|
|
1364
|
+
"--quiet",
|
|
1365
|
+
"--project",
|
|
1366
|
+
bundledProject,
|
|
1367
|
+
"python",
|
|
1368
|
+
target,
|
|
1369
|
+
...args
|
|
1370
|
+
];
|
|
1371
|
+
return {
|
|
1372
|
+
command: "uv",
|
|
1373
|
+
commandArgs,
|
|
1374
|
+
displayCommand: ["uv", ...commandArgs]
|
|
1375
|
+
};
|
|
999
1376
|
}
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1377
|
+
function withBundledPythonEnvironment(env) {
|
|
1378
|
+
return {
|
|
1379
|
+
...env,
|
|
1380
|
+
UV_PROJECT_ENVIRONMENT: BUNDLED_PYTHON_ENVIRONMENT
|
|
1381
|
+
};
|
|
1004
1382
|
}
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1383
|
+
function buildFoundationPythonCommand(entrypoint, args = []) {
|
|
1384
|
+
const target = entrypoint === "-c" ? entrypoint : join4(foundationProject, "src", entrypoint);
|
|
1385
|
+
const commandArgs = [
|
|
1386
|
+
"run",
|
|
1387
|
+
"--frozen",
|
|
1388
|
+
"--quiet",
|
|
1389
|
+
"--project",
|
|
1390
|
+
foundationProject,
|
|
1391
|
+
"python",
|
|
1392
|
+
target,
|
|
1393
|
+
...args
|
|
1394
|
+
];
|
|
1395
|
+
return {
|
|
1396
|
+
command: "uv",
|
|
1397
|
+
commandArgs,
|
|
1398
|
+
displayCommand: ["uv", ...commandArgs]
|
|
1399
|
+
};
|
|
1008
1400
|
}
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1401
|
+
function withFoundationPythonEnvironment(env) {
|
|
1402
|
+
return {
|
|
1403
|
+
...env,
|
|
1404
|
+
UV_PROJECT_ENVIRONMENT: FOUNDATION_PYTHON_ENVIRONMENT
|
|
1405
|
+
};
|
|
1014
1406
|
}
|
|
1015
|
-
|
|
1407
|
+
var ProcessCancelledError = class extends Error {
|
|
1408
|
+
constructor(message = "Process cancelled.") {
|
|
1409
|
+
super(message);
|
|
1410
|
+
this.name = "ProcessCancelledError";
|
|
1411
|
+
}
|
|
1412
|
+
};
|
|
1413
|
+
async function openProcessLog(path, exclusive, append) {
|
|
1414
|
+
const flags = constants.O_WRONLY | constants.O_CREAT | (exclusive ? constants.O_EXCL : append ? constants.O_APPEND : constants.O_TRUNC) | (process.platform === "win32" ? 0 : constants.O_NOFOLLOW);
|
|
1415
|
+
const handle = await open2(path, flags, 384);
|
|
1016
1416
|
try {
|
|
1017
|
-
|
|
1018
|
-
return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
1417
|
+
return handle.createWriteStream({ autoClose: true });
|
|
1019
1418
|
} catch (error) {
|
|
1020
|
-
|
|
1419
|
+
await handle.close();
|
|
1021
1420
|
throw error;
|
|
1022
1421
|
}
|
|
1023
1422
|
}
|
|
1024
|
-
function
|
|
1025
|
-
if (
|
|
1026
|
-
|
|
1423
|
+
async function runLoggedProcess(args) {
|
|
1424
|
+
if (args.logPath) await mkdir2(dirname4(args.logPath), { recursive: true });
|
|
1425
|
+
const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false, args.appendLog ?? false) : null;
|
|
1426
|
+
let stderr = "";
|
|
1427
|
+
try {
|
|
1428
|
+
const exitCode = await new Promise((resolvePromise, reject) => {
|
|
1429
|
+
const child = spawn(args.command, args.commandArgs, {
|
|
1430
|
+
cwd: args.cwd ? resolve6(args.cwd) : process.cwd(),
|
|
1431
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1432
|
+
env: args.env ?? process.env,
|
|
1433
|
+
detached: process.platform !== "win32"
|
|
1434
|
+
});
|
|
1435
|
+
let timedOut = false;
|
|
1436
|
+
let cancelled = false;
|
|
1437
|
+
let cancellationError;
|
|
1438
|
+
let cancellationCheckRunning = false;
|
|
1439
|
+
let forceKillTimer = null;
|
|
1440
|
+
const killProcessGroup = (signal) => {
|
|
1441
|
+
if (child.pid && process.platform !== "win32") {
|
|
1442
|
+
try {
|
|
1443
|
+
process.kill(-child.pid, signal);
|
|
1444
|
+
return;
|
|
1445
|
+
} catch {
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
child.kill(signal);
|
|
1449
|
+
};
|
|
1450
|
+
const requestStop = (signal = "SIGTERM") => {
|
|
1451
|
+
killProcessGroup(signal);
|
|
1452
|
+
if (!forceKillTimer) {
|
|
1453
|
+
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), args.shutdownGraceMs ?? 5e3);
|
|
1454
|
+
forceKillTimer.unref();
|
|
1455
|
+
}
|
|
1456
|
+
};
|
|
1457
|
+
const onSigint = () => {
|
|
1458
|
+
cancelled = true;
|
|
1459
|
+
requestStop("SIGINT");
|
|
1460
|
+
};
|
|
1461
|
+
const onSigterm = () => {
|
|
1462
|
+
cancelled = true;
|
|
1463
|
+
requestStop("SIGTERM");
|
|
1464
|
+
};
|
|
1465
|
+
process.once("SIGINT", onSigint);
|
|
1466
|
+
process.once("SIGTERM", onSigterm);
|
|
1467
|
+
const timer = args.timeoutMs ? setTimeout(() => {
|
|
1468
|
+
timedOut = true;
|
|
1469
|
+
requestStop();
|
|
1470
|
+
}, args.timeoutMs) : null;
|
|
1471
|
+
const cancellationTimer = args.shouldCancel ? setInterval(() => {
|
|
1472
|
+
if (cancellationCheckRunning || cancelled) return;
|
|
1473
|
+
cancellationCheckRunning = true;
|
|
1474
|
+
Promise.resolve(args.shouldCancel?.()).then((requested) => {
|
|
1475
|
+
if (!requested || cancelled) return;
|
|
1476
|
+
cancelled = true;
|
|
1477
|
+
requestStop();
|
|
1478
|
+
}).catch((error) => {
|
|
1479
|
+
cancellationError = error;
|
|
1480
|
+
requestStop();
|
|
1481
|
+
}).finally(() => {
|
|
1482
|
+
cancellationCheckRunning = false;
|
|
1483
|
+
});
|
|
1484
|
+
}, args.cancelPollMs ?? 250) : null;
|
|
1485
|
+
cancellationTimer?.unref();
|
|
1486
|
+
const clearProcessTimers = () => {
|
|
1487
|
+
if (timer) clearTimeout(timer);
|
|
1488
|
+
if (cancellationTimer) clearInterval(cancellationTimer);
|
|
1489
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
1490
|
+
process.off("SIGINT", onSigint);
|
|
1491
|
+
process.off("SIGTERM", onSigterm);
|
|
1492
|
+
};
|
|
1493
|
+
if (logStream) {
|
|
1494
|
+
child.stdout.pipe(logStream, { end: false });
|
|
1495
|
+
child.stderr.pipe(logStream, { end: false });
|
|
1496
|
+
}
|
|
1497
|
+
forwardStreamLines(child.stdout, (line) => {
|
|
1498
|
+
args.onLine?.(line, "stdout");
|
|
1499
|
+
if (args.reporter?.verbose) {
|
|
1500
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stdout", message: line }));
|
|
1501
|
+
}
|
|
1502
|
+
});
|
|
1503
|
+
forwardStreamLines(child.stderr, (line) => {
|
|
1504
|
+
stderr += `${line}
|
|
1505
|
+
`;
|
|
1506
|
+
args.onLine?.(line, "stderr");
|
|
1507
|
+
if (args.reporter?.verbose) {
|
|
1508
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stderr", message: line }));
|
|
1509
|
+
}
|
|
1510
|
+
});
|
|
1511
|
+
child.on("error", (error) => {
|
|
1512
|
+
clearProcessTimers();
|
|
1513
|
+
reject(timedOut ? new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`) : error);
|
|
1514
|
+
});
|
|
1515
|
+
child.on("close", (code) => {
|
|
1516
|
+
if (timedOut || cancelled || cancellationError || args.terminateProcessGroupOnExit) {
|
|
1517
|
+
killProcessGroup("SIGKILL");
|
|
1518
|
+
}
|
|
1519
|
+
clearProcessTimers();
|
|
1520
|
+
if (timedOut) {
|
|
1521
|
+
reject(new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`));
|
|
1522
|
+
return;
|
|
1523
|
+
}
|
|
1524
|
+
if (cancellationError) {
|
|
1525
|
+
reject(cancellationError);
|
|
1526
|
+
return;
|
|
1527
|
+
}
|
|
1528
|
+
if (cancelled) {
|
|
1529
|
+
reject(new ProcessCancelledError(`${args.stage} was cancelled.`));
|
|
1530
|
+
return;
|
|
1531
|
+
}
|
|
1532
|
+
resolvePromise(code ?? 1);
|
|
1533
|
+
});
|
|
1534
|
+
});
|
|
1535
|
+
return { exitCode, stderr };
|
|
1536
|
+
} finally {
|
|
1537
|
+
if (logStream) {
|
|
1538
|
+
await new Promise((resolveEnd) => {
|
|
1539
|
+
logStream.end(resolveEnd);
|
|
1540
|
+
});
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1027
1543
|
}
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1544
|
+
|
|
1545
|
+
// src/local-runtime/host-inventory.ts
|
|
1546
|
+
async function runCommand(command, args, options = {}) {
|
|
1547
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
1548
|
+
let interrupted;
|
|
1549
|
+
const result = await new Promise((resolveResult) => {
|
|
1550
|
+
let settled = false;
|
|
1551
|
+
let stdout = "";
|
|
1552
|
+
let stderr = "";
|
|
1553
|
+
let stopError;
|
|
1554
|
+
let forceKillTimer = null;
|
|
1555
|
+
const child = spawn2(command, args, {
|
|
1556
|
+
cwd: options.cwd,
|
|
1557
|
+
env: options.env ?? process.env,
|
|
1558
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1559
|
+
detached: process.platform !== "win32"
|
|
1560
|
+
});
|
|
1561
|
+
const killProcessGroup = (signal) => {
|
|
1562
|
+
if (child.pid && process.platform !== "win32") {
|
|
1563
|
+
try {
|
|
1564
|
+
process.kill(-child.pid, signal);
|
|
1565
|
+
return;
|
|
1566
|
+
} catch {
|
|
1567
|
+
}
|
|
1568
|
+
}
|
|
1569
|
+
child.kill(signal);
|
|
1570
|
+
};
|
|
1571
|
+
const requestStop = (signal = "SIGTERM") => {
|
|
1572
|
+
killProcessGroup(signal);
|
|
1573
|
+
if (!forceKillTimer) {
|
|
1574
|
+
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), 5e3);
|
|
1575
|
+
forceKillTimer.unref();
|
|
1576
|
+
}
|
|
1577
|
+
};
|
|
1578
|
+
const onSigint = () => {
|
|
1579
|
+
interrupted = "interrupted by SIGINT";
|
|
1580
|
+
stopError = interrupted;
|
|
1581
|
+
requestStop("SIGINT");
|
|
1582
|
+
};
|
|
1583
|
+
const onSigterm = () => {
|
|
1584
|
+
interrupted = "interrupted by SIGTERM";
|
|
1585
|
+
stopError = interrupted;
|
|
1586
|
+
requestStop("SIGTERM");
|
|
1587
|
+
};
|
|
1588
|
+
const finish = (result2) => {
|
|
1589
|
+
if (settled) return;
|
|
1590
|
+
settled = true;
|
|
1591
|
+
clearTimeout(timer);
|
|
1592
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
1593
|
+
process.off("SIGINT", onSigint);
|
|
1594
|
+
process.off("SIGTERM", onSigterm);
|
|
1595
|
+
resolveResult(result2);
|
|
1596
|
+
};
|
|
1597
|
+
const timer = setTimeout(() => {
|
|
1598
|
+
stopError = `timed out after ${timeoutMs}ms`;
|
|
1599
|
+
requestStop();
|
|
1600
|
+
}, timeoutMs);
|
|
1601
|
+
timer.unref();
|
|
1602
|
+
process.once("SIGINT", onSigint);
|
|
1603
|
+
process.once("SIGTERM", onSigterm);
|
|
1604
|
+
child.stdout.setEncoding("utf8");
|
|
1605
|
+
child.stderr.setEncoding("utf8");
|
|
1606
|
+
child.stdout.on("data", (chunk) => {
|
|
1607
|
+
stdout += chunk;
|
|
1608
|
+
});
|
|
1609
|
+
child.stderr.on("data", (chunk) => {
|
|
1610
|
+
stderr += chunk;
|
|
1611
|
+
});
|
|
1612
|
+
child.on("error", (error) => {
|
|
1613
|
+
finish({ code: null, stdout, stderr, error: error.message });
|
|
1614
|
+
});
|
|
1615
|
+
child.on("close", (code) => {
|
|
1616
|
+
finish({ code: stopError ? null : code, stdout, stderr, error: stopError });
|
|
1617
|
+
});
|
|
1618
|
+
});
|
|
1619
|
+
if (interrupted) throw new Error(interrupted);
|
|
1620
|
+
return result;
|
|
1031
1621
|
}
|
|
1032
|
-
function
|
|
1033
|
-
|
|
1034
|
-
return compareIds(left, right);
|
|
1622
|
+
function firstLine(value) {
|
|
1623
|
+
return value.trim().split(/\r?\n/)[0] ?? "";
|
|
1035
1624
|
}
|
|
1036
|
-
function
|
|
1037
|
-
|
|
1625
|
+
function parseMiB(value) {
|
|
1626
|
+
if (!value) return void 0;
|
|
1627
|
+
const parsed = Number(value.replace(/[^\d.]/g, ""));
|
|
1628
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
1629
|
+
return Math.round(parsed * 1024 * 1024);
|
|
1038
1630
|
}
|
|
1039
|
-
function
|
|
1040
|
-
|
|
1041
|
-
const resolvedCandidate = resolve4(candidate);
|
|
1042
|
-
return resolvedCandidate === resolvedRoot || resolvedCandidate.startsWith(`${resolvedRoot}${sep2}`);
|
|
1631
|
+
function isNvidiaSmiDecorativeLine(line) {
|
|
1632
|
+
return /^(?:mon|tue|wed|thu|fri|sat|sun)\b/i.test(line) || line.startsWith("+---") || line.startsWith("|");
|
|
1043
1633
|
}
|
|
1044
|
-
function
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
const
|
|
1054
|
-
const
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1634
|
+
function gpuFromName(name, index, extras = {}) {
|
|
1635
|
+
return {
|
|
1636
|
+
index,
|
|
1637
|
+
name,
|
|
1638
|
+
unified_memory: /spark|gb10/i.test(name),
|
|
1639
|
+
...extras
|
|
1640
|
+
};
|
|
1641
|
+
}
|
|
1642
|
+
function parseNvidiaGpus(stdout) {
|
|
1643
|
+
const gpus = [];
|
|
1644
|
+
for (const line of stdout.split(/\r?\n/).map((item) => item.trim()).filter(Boolean)) {
|
|
1645
|
+
if (isNvidiaSmiDecorativeLine(line)) continue;
|
|
1646
|
+
const parts = line.split(",").map((part) => part.trim());
|
|
1647
|
+
if (parts[0] && /^\d+$/.test(parts[0]) && parts.length >= 2) {
|
|
1648
|
+
const name = parts[1] || "NVIDIA GPU";
|
|
1649
|
+
gpus.push(gpuFromName(name, Number(parts[0]), {
|
|
1650
|
+
driver_version: parts[2] || void 0,
|
|
1651
|
+
memory_total_bytes: parseMiB(parts[3]),
|
|
1652
|
+
memory_free_bytes: parseMiB(parts[4]),
|
|
1653
|
+
compute_cap: parts[5] || void 0
|
|
1654
|
+
}));
|
|
1655
|
+
continue;
|
|
1656
|
+
}
|
|
1657
|
+
if (parts.length >= 2 && parts[0] && !/^(?:NVIDIA-SMI|Fail|Error)/i.test(parts[0])) {
|
|
1658
|
+
const name = parts[0];
|
|
1659
|
+
gpus.push(gpuFromName(name, gpus.length, {
|
|
1660
|
+
driver_version: parts[1] || void 0,
|
|
1661
|
+
memory_total_bytes: parseMiB(parts[2]),
|
|
1662
|
+
memory_free_bytes: parseMiB(parts[3])
|
|
1663
|
+
}));
|
|
1664
|
+
}
|
|
1066
1665
|
}
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1666
|
+
return gpus;
|
|
1667
|
+
}
|
|
1668
|
+
function parseNvidiaTableGpus(stdout) {
|
|
1669
|
+
const gpus = [];
|
|
1670
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
1671
|
+
const match = line.match(/^\|\s+(\d+)\s+(.+?)\s+(?:On|Off|N\/A)\s+\|/);
|
|
1672
|
+
if (!match) continue;
|
|
1673
|
+
const name = (match[2] ?? "").trim();
|
|
1674
|
+
if (!name || /^NVIDIA-SMI/i.test(name)) continue;
|
|
1675
|
+
gpus.push(gpuFromName(name, Number(match[1])));
|
|
1676
|
+
}
|
|
1677
|
+
return gpus;
|
|
1678
|
+
}
|
|
1679
|
+
var NVIDIA_CSV_QUERIES = [
|
|
1680
|
+
["--query-gpu=index,name,driver_version,memory.total,memory.free", "--format=csv,noheader,nounits"],
|
|
1681
|
+
["--query-gpu=name,driver_version,memory.total,memory.free", "--format=csv,noheader,nounits"],
|
|
1682
|
+
["--query-gpu=name,driver_version", "--format=csv,noheader"]
|
|
1683
|
+
];
|
|
1684
|
+
async function queryNvidiaGpus(env) {
|
|
1685
|
+
let last;
|
|
1686
|
+
for (const args of NVIDIA_CSV_QUERIES) {
|
|
1687
|
+
const result = await runCommand("nvidia-smi", args, { env, timeoutMs: 1e4 });
|
|
1688
|
+
last = result;
|
|
1689
|
+
if (result.code !== 0) continue;
|
|
1690
|
+
const gpus2 = parseNvidiaGpus(result.stdout);
|
|
1691
|
+
if (gpus2.length > 0) return { result, gpus: gpus2 };
|
|
1692
|
+
}
|
|
1693
|
+
const table = await runCommand("nvidia-smi", [], { env, timeoutMs: 1e4 });
|
|
1694
|
+
const gpus = table.code === 0 ? parseNvidiaTableGpus(table.stdout) : [];
|
|
1695
|
+
return { result: table.code === 0 || !last ? table : last, gpus };
|
|
1696
|
+
}
|
|
1697
|
+
async function diskCheck(name, path) {
|
|
1698
|
+
const resolvedPath = resolve7(path);
|
|
1699
|
+
try {
|
|
1700
|
+
await access(resolvedPath, constants2.R_OK);
|
|
1701
|
+
const fs = await statfs(resolvedPath);
|
|
1702
|
+
const freeBytes = Number(fs.bavail) * Number(fs.bsize);
|
|
1703
|
+
return {
|
|
1704
|
+
name,
|
|
1705
|
+
path: resolvedPath,
|
|
1706
|
+
free_bytes: freeBytes,
|
|
1707
|
+
ok: true,
|
|
1708
|
+
message: `${resolvedPath} (${Math.round(freeBytes / 1024 ** 3 * 10) / 10} GiB available)`
|
|
1709
|
+
};
|
|
1710
|
+
} catch (error) {
|
|
1711
|
+
let probe = resolvedPath;
|
|
1712
|
+
for (let i = 0; i < 6; i += 1) {
|
|
1713
|
+
const parent = dirname5(probe);
|
|
1714
|
+
if (parent === probe) break;
|
|
1715
|
+
probe = parent;
|
|
1716
|
+
try {
|
|
1717
|
+
const fs = await statfs(probe);
|
|
1718
|
+
const freeBytes = Number(fs.bavail) * Number(fs.bsize);
|
|
1719
|
+
return {
|
|
1720
|
+
name,
|
|
1721
|
+
path: resolvedPath,
|
|
1722
|
+
free_bytes: freeBytes,
|
|
1723
|
+
ok: true,
|
|
1724
|
+
message: `${resolvedPath} not created yet; ${probe} has ${Math.round(freeBytes / 1024 ** 3 * 10) / 10} GiB available`
|
|
1725
|
+
};
|
|
1726
|
+
} catch {
|
|
1727
|
+
continue;
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
return {
|
|
1731
|
+
name,
|
|
1732
|
+
path: resolvedPath,
|
|
1733
|
+
ok: false,
|
|
1734
|
+
message: `${resolvedPath}: ${error instanceof Error ? error.message : String(error)}`
|
|
1735
|
+
};
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
function pythonProbeSource() {
|
|
1739
|
+
return [
|
|
1740
|
+
"import json",
|
|
1741
|
+
"import torch, transformers, peft, huggingface_hub",
|
|
1742
|
+
"payload = {'python_ok': True, 'torch': torch.__version__, 'transformers': transformers.__version__, 'peft': getattr(peft, '__version__', None), 'cuda_available': bool(torch.cuda.is_available())}",
|
|
1743
|
+
"cuda = torch.cuda.is_available()",
|
|
1744
|
+
"payload.update({'cuda_device': torch.cuda.get_device_name(0), 'compute_capability': list(torch.cuda.get_device_capability(0)), 'total_memory_bytes': torch.cuda.get_device_properties(0).total_memory, 'bf16_supported': bool(torch.cuda.is_bf16_supported())} if cuda else {})",
|
|
1745
|
+
"print(json.dumps(payload))"
|
|
1746
|
+
].join("; ");
|
|
1747
|
+
}
|
|
1748
|
+
function parsePythonProbeJson(text) {
|
|
1749
|
+
const line = [...text.split(/\r?\n/)].reverse().find((item) => item.trim().startsWith("{"));
|
|
1750
|
+
if (!line) return void 0;
|
|
1751
|
+
try {
|
|
1752
|
+
const parsed = JSON.parse(line);
|
|
1753
|
+
if (parsed.python_ok !== true) return void 0;
|
|
1754
|
+
return {
|
|
1755
|
+
ok: true,
|
|
1756
|
+
message: line.trim(),
|
|
1757
|
+
torch: typeof parsed.torch === "string" ? parsed.torch : void 0,
|
|
1758
|
+
transformers: typeof parsed.transformers === "string" ? parsed.transformers : void 0,
|
|
1759
|
+
peft: typeof parsed.peft === "string" ? parsed.peft : void 0,
|
|
1760
|
+
cuda_available: parsed.cuda_available === true,
|
|
1761
|
+
cuda_device: typeof parsed.cuda_device === "string" ? parsed.cuda_device : void 0,
|
|
1762
|
+
compute_capability: Array.isArray(parsed.compute_capability) ? parsed.compute_capability.filter((value) => typeof value === "number") : void 0,
|
|
1763
|
+
total_memory_bytes: typeof parsed.total_memory_bytes === "number" ? parsed.total_memory_bytes : void 0,
|
|
1764
|
+
bf16_supported: parsed.bf16_supported === true
|
|
1765
|
+
};
|
|
1766
|
+
} catch {
|
|
1767
|
+
return void 0;
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
async function collectPythonRuntime(config, env) {
|
|
1771
|
+
const entrypoint = buildBundledPythonCommand("-c", [pythonProbeSource()]);
|
|
1772
|
+
const childEnv = withBundledPythonEnvironment(
|
|
1773
|
+
withHuggingFaceCacheEnvironment(
|
|
1774
|
+
minimalMachineLearningEnvironment(env),
|
|
1775
|
+
config?.paths.modelCache
|
|
1776
|
+
)
|
|
1777
|
+
);
|
|
1778
|
+
const result = await runCommand(entrypoint.command, entrypoint.commandArgs, {
|
|
1779
|
+
env: childEnv,
|
|
1780
|
+
timeoutMs: 18e5
|
|
1781
|
+
});
|
|
1782
|
+
if (result.code === 0) {
|
|
1783
|
+
return parsePythonProbeJson(result.stdout) ?? {
|
|
1784
|
+
ok: true,
|
|
1785
|
+
message: firstLine(result.stdout) || "Python runtime probe succeeded"
|
|
1786
|
+
};
|
|
1787
|
+
}
|
|
1788
|
+
return {
|
|
1789
|
+
ok: false,
|
|
1790
|
+
message: result.error ?? (firstLine(result.stderr) || `uv exited ${result.code}`)
|
|
1791
|
+
};
|
|
1792
|
+
}
|
|
1793
|
+
async function collectHostInventory(options = {}) {
|
|
1794
|
+
const env = options.env ?? process.env;
|
|
1795
|
+
const config = options.config;
|
|
1796
|
+
const cwd = options.cwd ?? process.cwd();
|
|
1797
|
+
const nodeVersion = process.versions.node;
|
|
1798
|
+
const nodeMajor = Number(nodeVersion.split(".")[0]);
|
|
1799
|
+
const artifactRoot = resolve7(cwd, config?.artifactRoot ?? join5(cwd, DEFAULT_ARTIFACT_ROOT));
|
|
1800
|
+
const storeRoot = resolve7(config?.storeRoot ?? defaultStoreRoot(env));
|
|
1801
|
+
const modelCache = resolve7(
|
|
1802
|
+
config?.paths.modelCache ?? env.HF_HOME ?? join5(homedir3(), ".cache", "huggingface")
|
|
1803
|
+
);
|
|
1804
|
+
const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
|
|
1805
|
+
diskCheck("artifact-root", artifactRoot),
|
|
1806
|
+
diskCheck("store-root", storeRoot),
|
|
1807
|
+
diskCheck("model-cache", modelCache),
|
|
1808
|
+
runCommand("uv", ["--version"], { env, timeoutMs: 1e4 }),
|
|
1809
|
+
queryNvidiaGpus(env)
|
|
1810
|
+
]);
|
|
1811
|
+
const nvidia = nvidiaQueried.result;
|
|
1812
|
+
const gpus = nvidiaQueried.gpus;
|
|
1813
|
+
const inventory = {
|
|
1814
|
+
collected_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1815
|
+
quick: options.quick === true,
|
|
1816
|
+
node: {
|
|
1817
|
+
version: nodeVersion,
|
|
1818
|
+
major: nodeMajor,
|
|
1819
|
+
ok: nodeMajor >= 22
|
|
1820
|
+
},
|
|
1821
|
+
os: {
|
|
1822
|
+
platform: process.platform,
|
|
1823
|
+
arch: process.arch,
|
|
1824
|
+
type: type(),
|
|
1825
|
+
cpu_count: cpus().length,
|
|
1826
|
+
total_memory_bytes: totalmem(),
|
|
1827
|
+
free_memory_bytes: freemem()
|
|
1828
|
+
},
|
|
1829
|
+
uv: {
|
|
1830
|
+
ok: uvVersion.code === 0,
|
|
1831
|
+
version: uvVersion.code === 0 ? firstLine(uvVersion.stdout) : void 0,
|
|
1832
|
+
message: uvVersion.code === 0 ? firstLine(uvVersion.stdout) : uvVersion.error ?? (firstLine(uvVersion.stderr) || "uv is not available")
|
|
1833
|
+
},
|
|
1834
|
+
gpus,
|
|
1835
|
+
nvidia_smi: {
|
|
1836
|
+
ok: nvidia.code === 0,
|
|
1837
|
+
message: nvidia.code === 0 ? gpus[0] ? `${gpus[0].name}${gpus[0].memory_total_bytes ? ` (${Math.round(gpus[0].memory_total_bytes / 1024 ** 3)} GiB)` : ""}` : firstLine(nvidia.stdout) || "nvidia-smi reported a GPU" : nvidia.error ?? (firstLine(nvidia.stderr) || "nvidia-smi not available")
|
|
1838
|
+
},
|
|
1839
|
+
disks: [artifactDisk, storeDisk, cacheDisk]
|
|
1840
|
+
};
|
|
1841
|
+
if (options.quick !== true) {
|
|
1842
|
+
inventory.python = uvVersion.code === 0 ? await collectPythonRuntime(config, env) : { ok: false, message: "Python probe skipped because uv is not available" };
|
|
1843
|
+
}
|
|
1844
|
+
return inventory;
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1847
|
+
// src/local-runtime/hardware-snapshot.ts
|
|
1848
|
+
import { chmod, mkdir as mkdir3, readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
|
|
1849
|
+
import { dirname as dirname6, join as join6 } from "path";
|
|
1850
|
+
var HARDWARE_SNAPSHOT_VERSION = 1;
|
|
1851
|
+
var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
1852
|
+
function hardwareSnapshotPath(env = process.env) {
|
|
1853
|
+
return join6(getTunedTensorHome(env), "hardware.json");
|
|
1854
|
+
}
|
|
1855
|
+
async function readHardwareSnapshot(env = process.env) {
|
|
1856
|
+
const path = hardwareSnapshotPath(env);
|
|
1857
|
+
try {
|
|
1858
|
+
const parsed = JSON.parse(await readFile2(path, "utf8"));
|
|
1859
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return void 0;
|
|
1860
|
+
const snapshot = parsed;
|
|
1861
|
+
if (snapshot.version !== HARDWARE_SNAPSHOT_VERSION) return void 0;
|
|
1862
|
+
if (typeof snapshot.collected_at !== "string" || !snapshot.capabilities || !snapshot.inventory) {
|
|
1863
|
+
return void 0;
|
|
1864
|
+
}
|
|
1865
|
+
const age = Date.now() - Date.parse(snapshot.collected_at);
|
|
1866
|
+
const stale = !Number.isFinite(age) || age > HARDWARE_SNAPSHOT_TTL_MS || age < 0;
|
|
1867
|
+
return { ...snapshot, stale, path };
|
|
1868
|
+
} catch (error) {
|
|
1869
|
+
if (error.code === "ENOENT") return void 0;
|
|
1870
|
+
return void 0;
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
async function writeHardwareSnapshot(snapshot, env = process.env) {
|
|
1874
|
+
const path = hardwareSnapshotPath(env);
|
|
1875
|
+
await mkdir3(dirname6(path), { recursive: true, mode: 448 });
|
|
1876
|
+
const gpu = snapshot.capabilities.gpu;
|
|
1877
|
+
const record = {
|
|
1878
|
+
version: HARDWARE_SNAPSHOT_VERSION,
|
|
1879
|
+
collected_at: snapshot.collected_at,
|
|
1880
|
+
quick: snapshot.quick,
|
|
1881
|
+
gpu_fingerprint: gpu ? { name: gpu.name, total_memory_bytes: gpu.memory_total_bytes } : void 0,
|
|
1882
|
+
inventory: snapshot.inventory,
|
|
1883
|
+
capabilities: snapshot.capabilities,
|
|
1884
|
+
summary: snapshot.summary ?? formatCapabilitySummary(snapshot.capabilities)
|
|
1885
|
+
};
|
|
1886
|
+
await writeFile2(path, `${JSON.stringify(record, null, 2)}
|
|
1887
|
+
`, { mode: 384 });
|
|
1888
|
+
await chmod(path, 384);
|
|
1889
|
+
return record;
|
|
1890
|
+
}
|
|
1891
|
+
function formatHostStatusLine(snapshot) {
|
|
1892
|
+
if (!snapshot) return "not inventoried (run tt hardware)";
|
|
1893
|
+
const stale = snapshot.stale ? `; stale \u2014 run tt hardware` : "";
|
|
1894
|
+
return `${snapshot.summary}${stale}`;
|
|
1895
|
+
}
|
|
1896
|
+
function formatAgentHostBlock(snapshot) {
|
|
1897
|
+
if (!snapshot) {
|
|
1898
|
+
return [
|
|
1899
|
+
"Host capability is not inventoried yet.",
|
|
1900
|
+
"When the user asks to examine this machine, GPU, VRAM, CUDA, or what can train, fine-tune, or infer here, call examine_hardware before recommending a base model, engine, or pipeline."
|
|
1901
|
+
].join(" ");
|
|
1902
|
+
}
|
|
1903
|
+
const stale = snapshot.stale ? " (stale \u2014 call examine_hardware to refresh)" : "";
|
|
1904
|
+
return [
|
|
1905
|
+
`Host capability (cached ${snapshot.collected_at}${stale}):`,
|
|
1906
|
+
snapshot.summary,
|
|
1907
|
+
"When the user wants to examine this host, GPU, or decide what can train/fine-tune/infer here, call examine_hardware before recommending a base model, engine, or pipeline.",
|
|
1908
|
+
"Recommend only workloads marked ready; mention tight as a caution. Never invent generic 7B/70B sizing."
|
|
1909
|
+
].join("\n");
|
|
1910
|
+
}
|
|
1911
|
+
|
|
1912
|
+
// src/local-runtime/hardware.ts
|
|
1913
|
+
async function assessHardware(options = {}) {
|
|
1914
|
+
const env = options.env ?? process.env;
|
|
1915
|
+
if (options.quick === true) {
|
|
1916
|
+
const existing = await readHardwareSnapshot(env);
|
|
1917
|
+
if (existing && !existing.quick && !existing.stale) {
|
|
1918
|
+
return {
|
|
1919
|
+
version: existing.version,
|
|
1920
|
+
collected_at: existing.collected_at,
|
|
1921
|
+
quick: existing.quick,
|
|
1922
|
+
gpu_fingerprint: existing.gpu_fingerprint,
|
|
1923
|
+
inventory: existing.inventory,
|
|
1924
|
+
capabilities: existing.capabilities,
|
|
1925
|
+
summary: existing.summary
|
|
1926
|
+
};
|
|
1927
|
+
}
|
|
1928
|
+
}
|
|
1929
|
+
const inventory = await collectHostInventory({
|
|
1930
|
+
config: options.config,
|
|
1931
|
+
quick: options.quick === true,
|
|
1932
|
+
env,
|
|
1933
|
+
cwd: options.cwd
|
|
1934
|
+
});
|
|
1935
|
+
const capabilities = evaluateCapabilities(inventory);
|
|
1936
|
+
return await writeHardwareSnapshot({
|
|
1937
|
+
collected_at: inventory.collected_at,
|
|
1938
|
+
quick: inventory.quick,
|
|
1939
|
+
inventory,
|
|
1940
|
+
capabilities,
|
|
1941
|
+
summary: formatCapabilitySummary(capabilities)
|
|
1942
|
+
}, env);
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
// src/local-runtime/store.ts
|
|
1946
|
+
import { appendFile, chmod as chmod2, copyFile as copyFile2, lstat as lstat3, mkdir as mkdir4, readdir as readdir3, readFile as readFile3, rename as rename2, rm, stat as stat2, writeFile as writeFile3 } from "fs/promises";
|
|
1947
|
+
import { dirname as dirname7, join as join7, resolve as resolve8, sep as sep2 } from "path";
|
|
1948
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
1949
|
+
function isTerminalRunState(state) {
|
|
1950
|
+
return state.status === "stage_completed" || state.status === "completed" || state.status === "failed" || state.status === "cancelled" && state.current_stage === "cancelled" && Boolean(state.completed_at);
|
|
1951
|
+
}
|
|
1952
|
+
function defaultLocalHome() {
|
|
1953
|
+
return defaultStoreRoot();
|
|
1954
|
+
}
|
|
1955
|
+
function localStorePaths(root) {
|
|
1956
|
+
return {
|
|
1957
|
+
root,
|
|
1958
|
+
specsDir: join7(root, "specs"),
|
|
1959
|
+
runsDir: join7(root, "runs"),
|
|
1960
|
+
modelsDir: join7(root, "models"),
|
|
1961
|
+
datasetsDir: join7(root, "datasets")
|
|
1962
|
+
};
|
|
1963
|
+
}
|
|
1964
|
+
async function exists(path) {
|
|
1965
|
+
try {
|
|
1966
|
+
await stat2(path);
|
|
1967
|
+
return true;
|
|
1968
|
+
} catch {
|
|
1969
|
+
return false;
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1972
|
+
async function ensurePrivateDirectory(path) {
|
|
1973
|
+
await mkdir4(path, { recursive: true, mode: 448 });
|
|
1974
|
+
const metadata = await lstat3(path);
|
|
1975
|
+
if (metadata.isSymbolicLink()) {
|
|
1976
|
+
throw new Error(`Refusing symbolic link in local store: ${path}`);
|
|
1977
|
+
}
|
|
1978
|
+
if (!metadata.isDirectory()) {
|
|
1979
|
+
throw new Error(`Expected local store directory: ${path}`);
|
|
1980
|
+
}
|
|
1981
|
+
await chmod2(path, 448);
|
|
1982
|
+
}
|
|
1983
|
+
async function hardenPrivateTree(path) {
|
|
1984
|
+
const metadata = await lstat3(path);
|
|
1985
|
+
if (metadata.isSymbolicLink()) {
|
|
1986
|
+
throw new Error(`Refusing symbolic link in local store: ${path}`);
|
|
1987
|
+
}
|
|
1988
|
+
if (metadata.isDirectory()) {
|
|
1989
|
+
await chmod2(path, 448);
|
|
1990
|
+
for (const entry of await readdir3(path)) {
|
|
1991
|
+
await hardenPrivateTree(join7(path, entry));
|
|
1992
|
+
}
|
|
1993
|
+
return;
|
|
1994
|
+
}
|
|
1995
|
+
if (metadata.isFile()) {
|
|
1996
|
+
await chmod2(path, 384);
|
|
1997
|
+
return;
|
|
1998
|
+
}
|
|
1999
|
+
throw new Error(`Refusing unsupported filesystem entry in local store: ${path}`);
|
|
2000
|
+
}
|
|
2001
|
+
async function hardenExistingLocalStore(root) {
|
|
2002
|
+
const resolvedRoot = resolve8(root);
|
|
2003
|
+
let metadata;
|
|
2004
|
+
try {
|
|
2005
|
+
metadata = await lstat3(resolvedRoot);
|
|
2006
|
+
} catch (error) {
|
|
2007
|
+
if (error.code === "ENOENT") return false;
|
|
2008
|
+
throw error;
|
|
2009
|
+
}
|
|
2010
|
+
if (metadata.isSymbolicLink()) {
|
|
2011
|
+
throw new Error(`Refusing symbolic link in local store: ${resolvedRoot}`);
|
|
2012
|
+
}
|
|
2013
|
+
if (!metadata.isDirectory()) {
|
|
2014
|
+
throw new Error(`Expected local store directory: ${resolvedRoot}`);
|
|
2015
|
+
}
|
|
2016
|
+
await hardenPrivateTree(resolvedRoot);
|
|
2017
|
+
return true;
|
|
2018
|
+
}
|
|
2019
|
+
async function writePrivateJsonAtomic(path, value) {
|
|
2020
|
+
await ensurePrivateDirectory(dirname7(path));
|
|
2021
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.${randomUUID2()}.tmp`;
|
|
2022
|
+
await writeFile3(tmp, `${JSON.stringify(value, null, 2)}
|
|
2023
|
+
`, {
|
|
2024
|
+
encoding: "utf8",
|
|
2025
|
+
mode: 384
|
|
2026
|
+
});
|
|
2027
|
+
await rename2(tmp, path);
|
|
2028
|
+
await chmod2(path, 384);
|
|
2029
|
+
}
|
|
2030
|
+
async function readJson2(path) {
|
|
2031
|
+
return JSON.parse(await readFile3(path, "utf8"));
|
|
2032
|
+
}
|
|
2033
|
+
async function appendJsonl(path, value) {
|
|
2034
|
+
await ensurePrivateDirectory(dirname7(path));
|
|
2035
|
+
await appendFile(path, `${JSON.stringify(value)}
|
|
2036
|
+
`, {
|
|
2037
|
+
encoding: "utf8",
|
|
2038
|
+
mode: 384
|
|
2039
|
+
});
|
|
2040
|
+
await chmod2(path, 384);
|
|
2041
|
+
}
|
|
2042
|
+
async function writePrivateText(path, value) {
|
|
2043
|
+
await ensurePrivateDirectory(dirname7(path));
|
|
2044
|
+
await writeFile3(path, value, { encoding: "utf8", mode: 384 });
|
|
2045
|
+
await chmod2(path, 384);
|
|
2046
|
+
}
|
|
2047
|
+
async function readJsonl(path) {
|
|
2048
|
+
if (!await exists(path)) return [];
|
|
2049
|
+
return (await readFile3(path, "utf8")).split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
|
|
2050
|
+
}
|
|
2051
|
+
async function copyIfExists(from, to) {
|
|
2052
|
+
if (!await exists(from)) return;
|
|
2053
|
+
await ensurePrivateDirectory(dirname7(to));
|
|
2054
|
+
await copyFile2(from, to);
|
|
2055
|
+
await chmod2(to, 384);
|
|
2056
|
+
}
|
|
2057
|
+
async function childDirectoryNames(path) {
|
|
2058
|
+
try {
|
|
2059
|
+
const entries = await readdir3(path, { withFileTypes: true });
|
|
2060
|
+
return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
2061
|
+
} catch (error) {
|
|
2062
|
+
if (error.code === "ENOENT") return [];
|
|
2063
|
+
throw error;
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
function compareIds(left, right) {
|
|
2067
|
+
if (left.id === right.id) return 0;
|
|
2068
|
+
return left.id < right.id ? -1 : 1;
|
|
2069
|
+
}
|
|
2070
|
+
function compareUpdatedAt(left, right) {
|
|
2071
|
+
if (left.updated_at !== right.updated_at) return left.updated_at > right.updated_at ? -1 : 1;
|
|
2072
|
+
return compareIds(left, right);
|
|
2073
|
+
}
|
|
2074
|
+
function compareCreatedAt(left, right) {
|
|
2075
|
+
if (left.created_at !== right.created_at) return left.created_at > right.created_at ? -1 : 1;
|
|
2076
|
+
return compareIds(left, right);
|
|
2077
|
+
}
|
|
2078
|
+
function findByIdOrPrefix(records, id) {
|
|
2079
|
+
return records.find((record) => record.id === id) ?? records.find((record) => record.id.startsWith(id));
|
|
2080
|
+
}
|
|
2081
|
+
function isPathInside(root, candidate) {
|
|
2082
|
+
const resolvedRoot = resolve8(root);
|
|
2083
|
+
const resolvedCandidate = resolve8(candidate);
|
|
2084
|
+
return resolvedCandidate === resolvedRoot || resolvedCandidate.startsWith(`${resolvedRoot}${sep2}`);
|
|
2085
|
+
}
|
|
2086
|
+
function createLocalStore(root = defaultLocalHome()) {
|
|
2087
|
+
const resolvedRoot = resolve8(root);
|
|
2088
|
+
const paths = localStorePaths(resolvedRoot);
|
|
2089
|
+
const runDir = (id) => join7(paths.runsDir, id);
|
|
2090
|
+
const runStatePath = (id) => join7(runDir(id), "state.json");
|
|
2091
|
+
const runEventsPath = (id) => join7(runDir(id), "progress.jsonl");
|
|
2092
|
+
const runRequestPath = (id) => join7(runDir(id), "request.json");
|
|
2093
|
+
const runReportPath = (id) => join7(runDir(id), "run-report.json");
|
|
2094
|
+
const cancellationPath = (id) => join7(runDir(id), "cancel.requested");
|
|
2095
|
+
const specDir = (id) => join7(paths.specsDir, id);
|
|
2096
|
+
const specPath = (id) => join7(specDir(id), "spec.json");
|
|
2097
|
+
const modelPath = (id) => join7(paths.modelsDir, id, "model.json");
|
|
2098
|
+
let ensurePromise;
|
|
2099
|
+
async function ensureOnce() {
|
|
2100
|
+
await ensurePrivateDirectory(paths.root);
|
|
2101
|
+
await Promise.all([
|
|
2102
|
+
ensurePrivateDirectory(paths.specsDir),
|
|
2103
|
+
ensurePrivateDirectory(paths.runsDir),
|
|
2104
|
+
ensurePrivateDirectory(paths.modelsDir),
|
|
2105
|
+
ensurePrivateDirectory(paths.datasetsDir)
|
|
2106
|
+
]);
|
|
2107
|
+
await hardenExistingLocalStore(paths.root);
|
|
2108
|
+
}
|
|
2109
|
+
async function ensure() {
|
|
2110
|
+
ensurePromise ??= ensureOnce().catch((error) => {
|
|
2111
|
+
ensurePromise = void 0;
|
|
2112
|
+
throw error;
|
|
2113
|
+
});
|
|
2114
|
+
await ensurePromise;
|
|
1073
2115
|
}
|
|
1074
2116
|
async function writeRunState(state) {
|
|
1075
2117
|
await writePrivateJsonAtomic(runStatePath(state.id), state);
|
|
@@ -1136,7 +2178,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1136
2178
|
...event
|
|
1137
2179
|
};
|
|
1138
2180
|
await appendJsonl(runEventsPath(state.id), row);
|
|
1139
|
-
await appendJsonl(
|
|
2181
|
+
await appendJsonl(join7(state.artifact_dir, "progress.jsonl"), {
|
|
1140
2182
|
at: row.occurred_at,
|
|
1141
2183
|
stage: row.stage,
|
|
1142
2184
|
status: row.status,
|
|
@@ -1167,7 +2209,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1167
2209
|
const previous = await getRun(runId);
|
|
1168
2210
|
if (isTerminalRunState(previous)) return previous;
|
|
1169
2211
|
await Promise.all([
|
|
1170
|
-
rm(
|
|
2212
|
+
rm(join7(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true }),
|
|
1171
2213
|
rm(runReportPath(previous.id), { force: true })
|
|
1172
2214
|
]);
|
|
1173
2215
|
if (previous.status === "cancelled" && previous.current_stage === "cancel_requested") return previous;
|
|
@@ -1320,6 +2362,13 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1320
2362
|
}
|
|
1321
2363
|
const request = await readJson2(runRequestPath(report.run_id));
|
|
1322
2364
|
const completedAt = report.created_at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
2365
|
+
if (!await exists(reportPath)) {
|
|
2366
|
+
throw new Error(`Run report not found: ${reportPath}`);
|
|
2367
|
+
}
|
|
2368
|
+
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
2369
|
+
if (await exists(cancellationPath(report.run_id))) {
|
|
2370
|
+
return preserveCancellationRequestState(report.run_id);
|
|
2371
|
+
}
|
|
1323
2372
|
const model = await registerModelRecord({
|
|
1324
2373
|
request,
|
|
1325
2374
|
training: report.training,
|
|
@@ -1327,7 +2376,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1327
2376
|
createdAt: completedAt
|
|
1328
2377
|
});
|
|
1329
2378
|
if (!model) {
|
|
1330
|
-
await rm(
|
|
2379
|
+
await rm(join7(paths.modelsDir, `local-${report.run_id}`), { recursive: true, force: true });
|
|
1331
2380
|
}
|
|
1332
2381
|
const previous = await getRun(report.run_id);
|
|
1333
2382
|
const state = {
|
|
@@ -1341,10 +2390,6 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1341
2390
|
completed_at: completedAt,
|
|
1342
2391
|
updated_at: completedAt
|
|
1343
2392
|
};
|
|
1344
|
-
if (!await exists(reportPath)) {
|
|
1345
|
-
throw new Error(`Run report not found: ${reportPath}`);
|
|
1346
|
-
}
|
|
1347
|
-
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
1348
2393
|
if (await exists(cancellationPath(report.run_id))) {
|
|
1349
2394
|
return preserveCancellationRequestState(report.run_id);
|
|
1350
2395
|
}
|
|
@@ -1365,7 +2410,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1365
2410
|
const previous = await getRun(runId);
|
|
1366
2411
|
if (options.report) await rm(runReportPath(previous.id), { force: true });
|
|
1367
2412
|
if (options.model) {
|
|
1368
|
-
await rm(
|
|
2413
|
+
await rm(join7(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true });
|
|
1369
2414
|
}
|
|
1370
2415
|
const state = {
|
|
1371
2416
|
...previous,
|
|
@@ -1442,9 +2487,9 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1442
2487
|
const state = await getRun(id);
|
|
1443
2488
|
const candidates = [];
|
|
1444
2489
|
if (state.report_path) {
|
|
1445
|
-
const reportPath =
|
|
2490
|
+
const reportPath = resolve8(state.report_path);
|
|
1446
2491
|
const allowedRoots = [resolvedRoot];
|
|
1447
|
-
if (state.artifact_dir) allowedRoots.push(
|
|
2492
|
+
if (state.artifact_dir) allowedRoots.push(resolve8(state.artifact_dir));
|
|
1448
2493
|
if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) {
|
|
1449
2494
|
candidates.push(reportPath);
|
|
1450
2495
|
}
|
|
@@ -1485,14 +2530,30 @@ var specSnapshotSchema = z.object({
|
|
|
1485
2530
|
}).strict();
|
|
1486
2531
|
var foundationHyperparametersSchema = z.object({
|
|
1487
2532
|
vocab_size: z.number().int().min(64).max(65536).default(256),
|
|
1488
|
-
max_chars: z.number().int().min(1e3).max(
|
|
2533
|
+
max_chars: z.number().int().min(1e3).max(1e12).default(2e4),
|
|
1489
2534
|
depth: z.number().int().min(1).max(64).default(2),
|
|
1490
2535
|
pretrain_steps: z.number().int().min(1).max(1e7).default(2),
|
|
1491
2536
|
finetune_steps: z.number().int().min(1).max(1e7).default(2),
|
|
1492
2537
|
rl_steps: z.number().int().min(0).max(1e7).default(0),
|
|
1493
2538
|
batch_size: z.number().int().min(1).max(1024).default(2),
|
|
1494
2539
|
sequence_length: z.number().int().min(16).max(8192).default(64),
|
|
1495
|
-
nproc_per_node: z.number().int().min(1).max(16).default(1)
|
|
2540
|
+
nproc_per_node: z.number().int().min(1).max(16).default(1),
|
|
2541
|
+
corpus_path: z.string().min(1).optional(),
|
|
2542
|
+
tokenizer_max_chars: z.number().int().min(1e3).max(1e12).optional(),
|
|
2543
|
+
validation_path: z.string().min(1).optional(),
|
|
2544
|
+
checkpoint_backup_dir: z.string().min(1).optional(),
|
|
2545
|
+
seed: z.number().int().min(0).max(2147483647).optional(),
|
|
2546
|
+
learning_rate: z.number().positive().max(1).optional(),
|
|
2547
|
+
weight_decay: z.number().min(0).max(10).optional(),
|
|
2548
|
+
warmup_steps: z.number().int().min(0).max(1e7).optional(),
|
|
2549
|
+
min_lr_ratio: z.number().min(0).max(1).optional(),
|
|
2550
|
+
gradient_accumulation_steps: z.number().int().min(1).max(1024).optional(),
|
|
2551
|
+
gradient_clip: z.number().positive().max(1e3).optional(),
|
|
2552
|
+
bf16: z.boolean().optional(),
|
|
2553
|
+
checkpoint_interval_steps: z.number().int().min(1).max(1e7).optional(),
|
|
2554
|
+
checkpoint_interval_seconds: z.number().int().min(1).max(86400).optional(),
|
|
2555
|
+
keep_checkpoints: z.number().int().min(1).max(100).optional(),
|
|
2556
|
+
log_interval_steps: z.number().int().min(1).max(1e6).optional()
|
|
1496
2557
|
}).strict();
|
|
1497
2558
|
var localFoundationSpecFileSchema = z.object({
|
|
1498
2559
|
engine: z.literal("foundation"),
|
|
@@ -1744,23 +2805,30 @@ function isFoundationSpecFile(spec) {
|
|
|
1744
2805
|
}
|
|
1745
2806
|
|
|
1746
2807
|
// src/local-runtime/local-project.ts
|
|
1747
|
-
import { createHash as
|
|
1748
|
-
import { readFile as
|
|
1749
|
-
import { dirname as
|
|
1750
|
-
import { homedir as
|
|
1751
|
-
import { mkdir as
|
|
2808
|
+
import { createHash as createHash3, randomUUID as randomUUID3 } from "crypto";
|
|
2809
|
+
import { readFile as readFile4, stat as stat3, writeFile as writeFile4 } from "fs/promises";
|
|
2810
|
+
import { dirname as dirname8, isAbsolute as isAbsolute3, relative as relative2, resolve as resolve9, sep as sep3 } from "path";
|
|
2811
|
+
import { homedir as homedir4 } from "os";
|
|
2812
|
+
import { mkdir as mkdir5 } from "fs/promises";
|
|
1752
2813
|
var DEFAULT_LOCAL_SPEC_PATH = "tunedtensor.json";
|
|
1753
2814
|
function resolveLocalReference(value, baseDirectory) {
|
|
1754
2815
|
if (typeof value !== "string" || !value || isAbsolute3(value) || /^[a-z][a-z0-9+.-]*:/i.test(value)) {
|
|
1755
2816
|
return value;
|
|
1756
2817
|
}
|
|
1757
|
-
if (value === "~") return
|
|
1758
|
-
if (value.startsWith("~/")) return
|
|
1759
|
-
return
|
|
2818
|
+
if (value === "~") return homedir4();
|
|
2819
|
+
if (value.startsWith("~/")) return resolve9(homedir4(), value.slice(2));
|
|
2820
|
+
return resolve9(baseDirectory, value);
|
|
2821
|
+
}
|
|
2822
|
+
function localPathsOverlap(left, right) {
|
|
2823
|
+
const contains = (parent, candidate) => {
|
|
2824
|
+
const relation = relative2(resolve9(parent), resolve9(candidate));
|
|
2825
|
+
return relation === "" || !isAbsolute3(relation) && relation !== ".." && !relation.startsWith(`..${sep3}`);
|
|
2826
|
+
};
|
|
2827
|
+
return contains(left, right) || contains(right, left);
|
|
1760
2828
|
}
|
|
1761
2829
|
function resolveLocalRunInputPaths(raw, inputPath) {
|
|
1762
2830
|
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw;
|
|
1763
|
-
const baseDirectory =
|
|
2831
|
+
const baseDirectory = dirname8(resolve9(inputPath));
|
|
1764
2832
|
const value = structuredClone(raw);
|
|
1765
2833
|
const dataset = value.dataset_prebuilt;
|
|
1766
2834
|
if (dataset && typeof dataset === "object" && !Array.isArray(dataset)) {
|
|
@@ -1769,6 +2837,13 @@ function resolveLocalRunInputPaths(raw, inputPath) {
|
|
|
1769
2837
|
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
1770
2838
|
}
|
|
1771
2839
|
}
|
|
2840
|
+
const foundation = value.foundation;
|
|
2841
|
+
if (foundation && typeof foundation === "object" && !Array.isArray(foundation)) {
|
|
2842
|
+
const fields = foundation;
|
|
2843
|
+
for (const key of ["corpus_path", "validation_path", "checkpoint_backup_dir"]) {
|
|
2844
|
+
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
1772
2847
|
return value;
|
|
1773
2848
|
}
|
|
1774
2849
|
async function exists2(path) {
|
|
@@ -1816,8 +2891,8 @@ async function initLocalSpecFile(args) {
|
|
|
1816
2891
|
nproc_per_node: 1
|
|
1817
2892
|
}
|
|
1818
2893
|
});
|
|
1819
|
-
await
|
|
1820
|
-
await
|
|
2894
|
+
await mkdir5(dirname8(args.outputPath), { recursive: true });
|
|
2895
|
+
await writeFile4(args.outputPath, `${JSON.stringify(spec2, null, 2)}
|
|
1821
2896
|
`, "utf8");
|
|
1822
2897
|
return spec2;
|
|
1823
2898
|
}
|
|
@@ -1848,8 +2923,8 @@ async function initLocalSpecFile(args) {
|
|
|
1848
2923
|
n_epochs: 1
|
|
1849
2924
|
}
|
|
1850
2925
|
});
|
|
1851
|
-
await
|
|
1852
|
-
await
|
|
2926
|
+
await mkdir5(dirname8(args.outputPath), { recursive: true });
|
|
2927
|
+
await writeFile4(args.outputPath, `${JSON.stringify(spec, null, 2)}
|
|
1853
2928
|
`, "utf8");
|
|
1854
2929
|
return spec;
|
|
1855
2930
|
}
|
|
@@ -1870,8 +2945,8 @@ async function initLocalRunnerConfigFile(args) {
|
|
|
1870
2945
|
timeoutMs: 18e5
|
|
1871
2946
|
}
|
|
1872
2947
|
};
|
|
1873
|
-
await
|
|
1874
|
-
await
|
|
2948
|
+
await mkdir5(dirname8(args.outputPath), { recursive: true });
|
|
2949
|
+
await writeFile4(args.outputPath, `${JSON.stringify(config, null, 2)}
|
|
1875
2950
|
`, "utf8");
|
|
1876
2951
|
return config;
|
|
1877
2952
|
}
|
|
@@ -1907,536 +2982,185 @@ function assertFoundationSpecReady(spec) {
|
|
|
1907
2982
|
}
|
|
1908
2983
|
function runRequestFromLocalSpec(spec, options = {}) {
|
|
1909
2984
|
if (isFoundationSpecFile(spec)) {
|
|
1910
|
-
throw new Error(
|
|
1911
|
-
"Foundation specs cannot be converted into a LoRA run request. Use `tt pipeline run --spec`."
|
|
1912
|
-
);
|
|
1913
|
-
}
|
|
1914
|
-
const {
|
|
1915
|
-
id,
|
|
1916
|
-
engine: _engine,
|
|
1917
|
-
hyperparameters,
|
|
1918
|
-
dataset_prebuilt: datasetPrebuilt,
|
|
1919
|
-
...specSnapshot
|
|
1920
|
-
} = spec;
|
|
1921
|
-
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
1922
|
-
return fineTuneRunRequestSchema.parse({
|
|
1923
|
-
run_id: options.runId ?? randomUUID3(),
|
|
1924
|
-
user_id: "local-user",
|
|
1925
|
-
behavior_spec_id: behaviorSpecId,
|
|
1926
|
-
run_number: 1,
|
|
1927
|
-
spec_snapshot: specSnapshot,
|
|
1928
|
-
hyperparameters,
|
|
1929
|
-
dataset_prebuilt: datasetPrebuilt
|
|
1930
|
-
});
|
|
1931
|
-
}
|
|
1932
|
-
function deterministicBehaviorSpecId(spec) {
|
|
1933
|
-
const namespace = Buffer.from("9f783f01340457bea061942004259253", "hex");
|
|
1934
|
-
const digest = createHash2("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
|
|
1935
|
-
digest[6] = digest[6] & 15 | 80;
|
|
1936
|
-
digest[8] = digest[8] & 63 | 128;
|
|
1937
|
-
const hex = digest.toString("hex");
|
|
1938
|
-
return [
|
|
1939
|
-
hex.slice(0, 8),
|
|
1940
|
-
hex.slice(8, 12),
|
|
1941
|
-
hex.slice(12, 16),
|
|
1942
|
-
hex.slice(16, 20),
|
|
1943
|
-
hex.slice(20)
|
|
1944
|
-
].join("-");
|
|
1945
|
-
}
|
|
1946
|
-
async function loadLocalRunInput(path, options = {}) {
|
|
1947
|
-
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile3(path, "utf8")), path);
|
|
1948
|
-
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
1949
|
-
if (request.success) {
|
|
1950
|
-
return { kind: "request", path, request: request.data };
|
|
1951
|
-
}
|
|
1952
|
-
const spec = localBehaviorSpecFileSchema.parse(raw);
|
|
1953
|
-
if (isFoundationSpecFile(spec)) {
|
|
1954
|
-
return { kind: "foundation-spec", path, spec };
|
|
1955
|
-
}
|
|
1956
|
-
return {
|
|
1957
|
-
kind: "spec",
|
|
1958
|
-
path,
|
|
1959
|
-
spec,
|
|
1960
|
-
request: runRequestFromLocalSpec(spec, options)
|
|
1961
|
-
};
|
|
1962
|
-
}
|
|
1963
|
-
|
|
1964
|
-
// src/local-runtime/dataset.ts
|
|
1965
|
-
import { readFile as readFile4 } from "fs/promises";
|
|
1966
|
-
async function loadNormalizedChatJsonl(path) {
|
|
1967
|
-
const text = await readFile4(path, "utf8");
|
|
1968
|
-
const rows = [];
|
|
1969
|
-
const jsonLines = [];
|
|
1970
|
-
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
1971
|
-
if (!line.trim()) continue;
|
|
1972
|
-
let value;
|
|
1973
|
-
try {
|
|
1974
|
-
value = JSON.parse(line);
|
|
1975
|
-
} catch (error) {
|
|
1976
|
-
throw new Error(`Invalid chat JSONL row ${index + 1}: malformed JSON`, { cause: error });
|
|
1977
|
-
}
|
|
1978
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1979
|
-
throw new Error(`Invalid chat JSONL row ${index + 1}: expected an object`);
|
|
1980
|
-
}
|
|
1981
|
-
const messages = value.messages;
|
|
1982
|
-
if (!Array.isArray(messages) || ![2, 3].includes(messages.length) || messages.some(
|
|
1983
|
-
(message) => !message || typeof message !== "object" || Array.isArray(message) || !["system", "user", "assistant"].includes(String(message.role)) || typeof message.content !== "string"
|
|
1984
|
-
)) {
|
|
1985
|
-
throw new Error(
|
|
1986
|
-
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one user message, and one assistant answer`
|
|
1987
|
-
);
|
|
1988
|
-
}
|
|
1989
|
-
const row = { messages };
|
|
1990
|
-
const offset = row.messages.length === 3 ? 1 : 0;
|
|
1991
|
-
if (offset === 1 && row.messages[0]?.role !== "system" || row.messages[offset]?.role !== "user" || !row.messages[offset]?.content.trim() || row.messages[offset + 1]?.role !== "assistant" || !row.messages[offset + 1]?.content.trim()) {
|
|
1992
|
-
throw new Error(
|
|
1993
|
-
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one non-empty user message, and one non-empty assistant answer`
|
|
1994
|
-
);
|
|
1995
|
-
}
|
|
1996
|
-
rows.push(row);
|
|
1997
|
-
jsonLines.push(JSON.stringify(row));
|
|
1998
|
-
}
|
|
1999
|
-
if (rows.length === 0) throw new Error(`Chat JSONL contains no examples: ${path}`);
|
|
2000
|
-
return {
|
|
2001
|
-
rows,
|
|
2002
|
-
jsonl: jsonLines.join("\n")
|
|
2003
|
-
};
|
|
2004
|
-
}
|
|
2005
|
-
async function normalizeChatJsonlForRelocation(path) {
|
|
2006
|
-
return (await loadNormalizedChatJsonl(path)).jsonl;
|
|
2007
|
-
}
|
|
2008
|
-
function buildSystemMessage(spec) {
|
|
2009
|
-
const parts = [];
|
|
2010
|
-
if (spec.system_prompt.trim()) parts.push(spec.system_prompt.trim());
|
|
2011
|
-
if (spec.guidelines.length > 0) {
|
|
2012
|
-
parts.push(`Guidelines:
|
|
2013
|
-
${spec.guidelines.map((guideline) => `- ${guideline}`).join("\n")}`);
|
|
2014
|
-
}
|
|
2015
|
-
if (spec.constraints.length > 0) {
|
|
2016
|
-
parts.push(`Constraints:
|
|
2017
|
-
${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
|
|
2018
|
-
}
|
|
2019
|
-
return parts.join("\n\n") || "Follow the demonstrated behavior.";
|
|
2020
|
-
}
|
|
2021
|
-
function exampleToChatRow(spec, example) {
|
|
2022
|
-
const system = buildSystemMessage(spec);
|
|
2023
|
-
return {
|
|
2024
|
-
messages: [
|
|
2025
|
-
{ role: "system", content: system },
|
|
2026
|
-
{ role: "user", content: example.input },
|
|
2027
|
-
{ role: "assistant", content: example.output }
|
|
2028
|
-
]
|
|
2029
|
-
};
|
|
2030
|
-
}
|
|
2031
|
-
function compileSpecToJsonl(spec) {
|
|
2032
|
-
return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
|
|
2033
|
-
}
|
|
2034
|
-
function examplesFromSpec(spec) {
|
|
2035
|
-
return spec.examples;
|
|
2036
|
-
}
|
|
2037
|
-
async function examplesFromChatJsonl(path) {
|
|
2038
|
-
const normalized = await loadNormalizedChatJsonl(path);
|
|
2039
|
-
const examples = [];
|
|
2040
|
-
for (const [index, row] of normalized.rows.entries()) {
|
|
2041
|
-
const user = row.messages.at(-2);
|
|
2042
|
-
const assistant = row.messages.at(-1);
|
|
2043
|
-
examples.push({
|
|
2044
|
-
input: user.content,
|
|
2045
|
-
output: assistant.content
|
|
2046
|
-
});
|
|
2047
|
-
}
|
|
2048
|
-
return examples;
|
|
2049
|
-
}
|
|
2050
|
-
async function evaluationSuiteFromChatJsonl(path, configuredSystem) {
|
|
2051
|
-
const normalized = await loadNormalizedChatJsonl(path);
|
|
2052
|
-
const inputIdentities = /* @__PURE__ */ new Set();
|
|
2053
|
-
for (const row of normalized.rows) {
|
|
2054
|
-
const input = row.messages.at(-2).content;
|
|
2055
|
-
const identity = input.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2056
|
-
if (inputIdentities.has(identity)) {
|
|
2057
|
-
throw new Error(
|
|
2058
|
-
"General regression data contains duplicate inputs; evaluation prompts must be unique."
|
|
2059
|
-
);
|
|
2060
|
-
}
|
|
2061
|
-
inputIdentities.add(identity);
|
|
2062
|
-
}
|
|
2063
|
-
const systems = new Set(
|
|
2064
|
-
normalized.rows.map((row) => row.messages.find((message) => message.role === "system")?.content).filter((value) => value !== void 0)
|
|
2065
|
-
);
|
|
2066
|
-
if (systems.size > 1) {
|
|
2067
|
-
throw new Error("General regression rows must use one consistent system message.");
|
|
2068
|
-
}
|
|
2069
|
-
const embeddedSystem = [...systems][0];
|
|
2070
|
-
if (configuredSystem !== void 0 && embeddedSystem !== void 0 && configuredSystem !== embeddedSystem) {
|
|
2071
|
-
throw new Error(
|
|
2072
|
-
"evaluation.generalRegression.systemPrompt must match the system message embedded in the dataset."
|
|
2073
|
-
);
|
|
2074
|
-
}
|
|
2075
|
-
return {
|
|
2076
|
-
examples: normalized.rows.map((row) => ({
|
|
2077
|
-
input: row.messages.at(-2).content,
|
|
2078
|
-
output: row.messages.at(-1).content
|
|
2079
|
-
})),
|
|
2080
|
-
system: configuredSystem ?? embeddedSystem ?? "You are a helpful assistant."
|
|
2081
|
-
};
|
|
2082
|
-
}
|
|
2083
|
-
|
|
2084
|
-
// src/local-runtime/run-reporter.ts
|
|
2085
|
-
function reportInBackground(action) {
|
|
2086
|
-
try {
|
|
2087
|
-
void Promise.resolve(action()).catch(() => void 0);
|
|
2088
|
-
} catch {
|
|
2089
|
-
}
|
|
2090
|
-
}
|
|
2091
|
-
function sanitizeLogLine(line) {
|
|
2092
|
-
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
2093
|
-
}
|
|
2094
|
-
function forwardStreamLines(stream, onLine) {
|
|
2095
|
-
let buffered = "";
|
|
2096
|
-
stream.setEncoding("utf8");
|
|
2097
|
-
stream.on("data", (chunk) => {
|
|
2098
|
-
buffered += chunk;
|
|
2099
|
-
const lines = buffered.split(/\r\n|\n|\r/);
|
|
2100
|
-
buffered = lines.pop() ?? "";
|
|
2101
|
-
for (const line of lines) {
|
|
2102
|
-
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
2103
|
-
}
|
|
2104
|
-
});
|
|
2105
|
-
stream.on("end", () => {
|
|
2106
|
-
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
2107
|
-
});
|
|
2108
|
-
}
|
|
2109
|
-
|
|
2110
|
-
// src/local-runtime/package-root.ts
|
|
2111
|
-
import { existsSync as existsSync2 } from "fs";
|
|
2112
|
-
import { dirname as dirname4, resolve as resolve6 } from "path";
|
|
2113
|
-
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
2114
|
-
function localRuntimePackageRoot(moduleUrl) {
|
|
2115
|
-
const moduleDirectory = dirname4(fileURLToPath2(moduleUrl));
|
|
2116
|
-
const candidates = [
|
|
2117
|
-
dirname4(moduleDirectory),
|
|
2118
|
-
dirname4(dirname4(moduleDirectory))
|
|
2119
|
-
];
|
|
2120
|
-
const root = candidates.find(
|
|
2121
|
-
(candidate) => existsSync2(resolve6(candidate, "package.json")) && existsSync2(resolve6(candidate, "training/adapter"))
|
|
2122
|
-
);
|
|
2123
|
-
if (!root) {
|
|
2124
|
-
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
2125
|
-
}
|
|
2126
|
-
return root;
|
|
2127
|
-
}
|
|
2128
|
-
|
|
2129
|
-
// src/local-runtime/prefetch.ts
|
|
2130
|
-
import { createHash as createHash4 } from "crypto";
|
|
2131
|
-
import { createReadStream as createReadStream2 } from "fs";
|
|
2132
|
-
import { lstat as lstat4, mkdir as mkdir5, readFile as readFile5, readdir as readdir4, realpath as realpath2, stat as stat4 } from "fs/promises";
|
|
2133
|
-
import { isAbsolute as isAbsolute4, join as join6, relative as relative2, resolve as resolve9, sep as sep3 } from "path";
|
|
2134
|
-
|
|
2135
|
-
// src/local-runtime/process-runner.ts
|
|
2136
|
-
import { createHash as createHash3 } from "crypto";
|
|
2137
|
-
import { spawn } from "child_process";
|
|
2138
|
-
import { createWriteStream, readFileSync } from "fs";
|
|
2139
|
-
import { mkdir as mkdir4 } from "fs/promises";
|
|
2140
|
-
import { dirname as dirname5, join as join4, resolve as resolve7 } from "path";
|
|
2141
|
-
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
2142
|
-
var bundledProject = join4(packageRoot, "training/adapter");
|
|
2143
|
-
var bundledRuntimeHash = (() => {
|
|
2144
|
-
const hash = createHash3("sha256");
|
|
2145
|
-
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
2146
|
-
hash.update(readFileSync(join4(bundledProject, name)));
|
|
2147
|
-
}
|
|
2148
|
-
return hash.digest("hex").slice(0, 20);
|
|
2149
|
-
})();
|
|
2150
|
-
var BUNDLED_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv", bundledRuntimeHash);
|
|
2151
|
-
var foundationProject = join4(packageRoot, "training/foundation");
|
|
2152
|
-
var foundationRuntimeHash = (() => {
|
|
2153
|
-
const hash = createHash3("sha256");
|
|
2154
|
-
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
2155
|
-
hash.update(readFileSync(join4(foundationProject, name)));
|
|
2156
|
-
}
|
|
2157
|
-
return hash.digest("hex").slice(0, 20);
|
|
2158
|
-
})();
|
|
2159
|
-
var FOUNDATION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-foundation", foundationRuntimeHash);
|
|
2160
|
-
function buildBundledPythonCommand(entrypoint, args = []) {
|
|
2161
|
-
const target = entrypoint === "-c" ? entrypoint : join4(bundledProject, "src", entrypoint);
|
|
2162
|
-
const commandArgs = [
|
|
2163
|
-
"run",
|
|
2164
|
-
"--frozen",
|
|
2165
|
-
"--quiet",
|
|
2166
|
-
"--project",
|
|
2167
|
-
bundledProject,
|
|
2168
|
-
"python",
|
|
2169
|
-
target,
|
|
2170
|
-
...args
|
|
2171
|
-
];
|
|
2172
|
-
return {
|
|
2173
|
-
command: "uv",
|
|
2174
|
-
commandArgs,
|
|
2175
|
-
displayCommand: ["uv", ...commandArgs]
|
|
2176
|
-
};
|
|
2177
|
-
}
|
|
2178
|
-
function withBundledPythonEnvironment(env) {
|
|
2179
|
-
return {
|
|
2180
|
-
...env,
|
|
2181
|
-
UV_PROJECT_ENVIRONMENT: BUNDLED_PYTHON_ENVIRONMENT
|
|
2182
|
-
};
|
|
2183
|
-
}
|
|
2184
|
-
function buildFoundationPythonCommand(entrypoint, args = []) {
|
|
2185
|
-
const commandArgs = [
|
|
2186
|
-
"run",
|
|
2187
|
-
"--frozen",
|
|
2188
|
-
"--quiet",
|
|
2189
|
-
"--project",
|
|
2190
|
-
foundationProject,
|
|
2191
|
-
"python",
|
|
2192
|
-
join4(foundationProject, "src", entrypoint),
|
|
2193
|
-
...args
|
|
2194
|
-
];
|
|
2195
|
-
return {
|
|
2196
|
-
command: "uv",
|
|
2197
|
-
commandArgs,
|
|
2198
|
-
displayCommand: ["uv", ...commandArgs]
|
|
2199
|
-
};
|
|
2200
|
-
}
|
|
2201
|
-
function withFoundationPythonEnvironment(env) {
|
|
2202
|
-
return {
|
|
2203
|
-
...env,
|
|
2204
|
-
UV_PROJECT_ENVIRONMENT: FOUNDATION_PYTHON_ENVIRONMENT
|
|
2205
|
-
};
|
|
2206
|
-
}
|
|
2207
|
-
var ProcessCancelledError = class extends Error {
|
|
2208
|
-
constructor(message = "Process cancelled.") {
|
|
2209
|
-
super(message);
|
|
2210
|
-
this.name = "ProcessCancelledError";
|
|
2211
|
-
}
|
|
2212
|
-
};
|
|
2213
|
-
async function openProcessLog(path, exclusive) {
|
|
2214
|
-
const stream = createWriteStream(path, {
|
|
2215
|
-
flags: exclusive ? "wx" : "w",
|
|
2216
|
-
mode: 384
|
|
2217
|
-
});
|
|
2218
|
-
await new Promise((resolveOpen, reject) => {
|
|
2219
|
-
const onOpen = () => {
|
|
2220
|
-
stream.off("error", onError);
|
|
2221
|
-
resolveOpen();
|
|
2222
|
-
};
|
|
2223
|
-
const onError = (error) => {
|
|
2224
|
-
stream.off("open", onOpen);
|
|
2225
|
-
reject(error);
|
|
2226
|
-
};
|
|
2227
|
-
stream.once("open", onOpen);
|
|
2228
|
-
stream.once("error", onError);
|
|
2229
|
-
});
|
|
2230
|
-
return stream;
|
|
2231
|
-
}
|
|
2232
|
-
async function runLoggedProcess(args) {
|
|
2233
|
-
if (args.logPath) await mkdir4(dirname5(args.logPath), { recursive: true });
|
|
2234
|
-
const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false) : null;
|
|
2235
|
-
let stderr = "";
|
|
2236
|
-
try {
|
|
2237
|
-
const exitCode = await new Promise((resolvePromise, reject) => {
|
|
2238
|
-
const child = spawn(args.command, args.commandArgs, {
|
|
2239
|
-
cwd: args.cwd ? resolve7(args.cwd) : process.cwd(),
|
|
2240
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
2241
|
-
env: args.env ?? process.env,
|
|
2242
|
-
detached: process.platform !== "win32"
|
|
2243
|
-
});
|
|
2244
|
-
let timedOut = false;
|
|
2245
|
-
let cancelled = false;
|
|
2246
|
-
let cancellationError;
|
|
2247
|
-
let cancellationCheckRunning = false;
|
|
2248
|
-
let forceKillTimer = null;
|
|
2249
|
-
const killProcessGroup = (signal) => {
|
|
2250
|
-
if (child.pid && process.platform !== "win32") {
|
|
2251
|
-
try {
|
|
2252
|
-
process.kill(-child.pid, signal);
|
|
2253
|
-
return;
|
|
2254
|
-
} catch {
|
|
2255
|
-
}
|
|
2256
|
-
}
|
|
2257
|
-
child.kill(signal);
|
|
2258
|
-
};
|
|
2259
|
-
const requestStop = (signal = "SIGTERM") => {
|
|
2260
|
-
killProcessGroup(signal);
|
|
2261
|
-
if (!forceKillTimer) {
|
|
2262
|
-
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), 5e3);
|
|
2263
|
-
forceKillTimer.unref();
|
|
2264
|
-
}
|
|
2265
|
-
};
|
|
2266
|
-
const onSigint = () => {
|
|
2267
|
-
cancelled = true;
|
|
2268
|
-
requestStop("SIGINT");
|
|
2269
|
-
};
|
|
2270
|
-
const onSigterm = () => {
|
|
2271
|
-
cancelled = true;
|
|
2272
|
-
requestStop("SIGTERM");
|
|
2273
|
-
};
|
|
2274
|
-
process.once("SIGINT", onSigint);
|
|
2275
|
-
process.once("SIGTERM", onSigterm);
|
|
2276
|
-
const timer = args.timeoutMs ? setTimeout(() => {
|
|
2277
|
-
timedOut = true;
|
|
2278
|
-
requestStop();
|
|
2279
|
-
}, args.timeoutMs) : null;
|
|
2280
|
-
const cancellationTimer = args.shouldCancel ? setInterval(() => {
|
|
2281
|
-
if (cancellationCheckRunning || cancelled) return;
|
|
2282
|
-
cancellationCheckRunning = true;
|
|
2283
|
-
Promise.resolve(args.shouldCancel?.()).then((requested) => {
|
|
2284
|
-
if (!requested || cancelled) return;
|
|
2285
|
-
cancelled = true;
|
|
2286
|
-
requestStop();
|
|
2287
|
-
}).catch((error) => {
|
|
2288
|
-
cancellationError = error;
|
|
2289
|
-
requestStop();
|
|
2290
|
-
}).finally(() => {
|
|
2291
|
-
cancellationCheckRunning = false;
|
|
2292
|
-
});
|
|
2293
|
-
}, args.cancelPollMs ?? 250) : null;
|
|
2294
|
-
cancellationTimer?.unref();
|
|
2295
|
-
const clearProcessTimers = () => {
|
|
2296
|
-
if (timer) clearTimeout(timer);
|
|
2297
|
-
if (cancellationTimer) clearInterval(cancellationTimer);
|
|
2298
|
-
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
2299
|
-
process.off("SIGINT", onSigint);
|
|
2300
|
-
process.off("SIGTERM", onSigterm);
|
|
2301
|
-
};
|
|
2302
|
-
if (logStream) {
|
|
2303
|
-
child.stdout.pipe(logStream, { end: false });
|
|
2304
|
-
child.stderr.pipe(logStream, { end: false });
|
|
2305
|
-
}
|
|
2306
|
-
forwardStreamLines(child.stdout, (line) => {
|
|
2307
|
-
args.onLine?.(line, "stdout");
|
|
2308
|
-
if (args.reporter?.verbose) {
|
|
2309
|
-
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stdout", message: line }));
|
|
2310
|
-
}
|
|
2311
|
-
});
|
|
2312
|
-
forwardStreamLines(child.stderr, (line) => {
|
|
2313
|
-
stderr += `${line}
|
|
2314
|
-
`;
|
|
2315
|
-
args.onLine?.(line, "stderr");
|
|
2316
|
-
if (args.reporter?.verbose) {
|
|
2317
|
-
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stderr", message: line }));
|
|
2318
|
-
}
|
|
2319
|
-
});
|
|
2320
|
-
child.on("error", (error) => {
|
|
2321
|
-
clearProcessTimers();
|
|
2322
|
-
reject(timedOut ? new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`) : error);
|
|
2323
|
-
});
|
|
2324
|
-
child.on("close", (code) => {
|
|
2325
|
-
if (timedOut || cancelled || cancellationError || args.terminateProcessGroupOnExit) {
|
|
2326
|
-
killProcessGroup("SIGKILL");
|
|
2327
|
-
}
|
|
2328
|
-
clearProcessTimers();
|
|
2329
|
-
if (timedOut) {
|
|
2330
|
-
reject(new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`));
|
|
2331
|
-
return;
|
|
2332
|
-
}
|
|
2333
|
-
if (cancellationError) {
|
|
2334
|
-
reject(cancellationError);
|
|
2335
|
-
return;
|
|
2336
|
-
}
|
|
2337
|
-
if (cancelled) {
|
|
2338
|
-
reject(new ProcessCancelledError(`${args.stage} was cancelled.`));
|
|
2339
|
-
return;
|
|
2340
|
-
}
|
|
2341
|
-
resolvePromise(code ?? 1);
|
|
2342
|
-
});
|
|
2343
|
-
});
|
|
2344
|
-
return { exitCode, stderr };
|
|
2345
|
-
} finally {
|
|
2346
|
-
if (logStream) {
|
|
2347
|
-
await new Promise((resolveEnd) => {
|
|
2348
|
-
logStream.end(resolveEnd);
|
|
2349
|
-
});
|
|
2350
|
-
}
|
|
2985
|
+
throw new Error(
|
|
2986
|
+
"Foundation specs cannot be converted into a LoRA run request. Use `tt pipeline run --spec`."
|
|
2987
|
+
);
|
|
2351
2988
|
}
|
|
2989
|
+
const {
|
|
2990
|
+
id,
|
|
2991
|
+
engine: _engine,
|
|
2992
|
+
hyperparameters,
|
|
2993
|
+
dataset_prebuilt: datasetPrebuilt,
|
|
2994
|
+
...specSnapshot
|
|
2995
|
+
} = spec;
|
|
2996
|
+
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
2997
|
+
return fineTuneRunRequestSchema.parse({
|
|
2998
|
+
run_id: options.runId ?? randomUUID3(),
|
|
2999
|
+
user_id: "local-user",
|
|
3000
|
+
behavior_spec_id: behaviorSpecId,
|
|
3001
|
+
run_number: 1,
|
|
3002
|
+
spec_snapshot: specSnapshot,
|
|
3003
|
+
hyperparameters,
|
|
3004
|
+
dataset_prebuilt: datasetPrebuilt
|
|
3005
|
+
});
|
|
2352
3006
|
}
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
const
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
"TMP",
|
|
2367
|
-
"LANG",
|
|
2368
|
-
"LANGUAGE",
|
|
2369
|
-
"LD_LIBRARY_PATH",
|
|
2370
|
-
"DYLD_LIBRARY_PATH",
|
|
2371
|
-
"XDG_CACHE_HOME",
|
|
2372
|
-
"SSL_CERT_FILE",
|
|
2373
|
-
"REQUESTS_CA_BUNDLE",
|
|
2374
|
-
"CURL_CA_BUNDLE",
|
|
2375
|
-
"NO_PROXY",
|
|
2376
|
-
"no_proxy",
|
|
2377
|
-
"UV_CACHE_DIR",
|
|
2378
|
-
"UV_PYTHON",
|
|
2379
|
-
"HF_HOME",
|
|
2380
|
-
"HF_HUB_CACHE",
|
|
2381
|
-
"HUGGINGFACE_HUB_CACHE",
|
|
2382
|
-
"HF_ENDPOINT",
|
|
2383
|
-
"HF_TOKEN",
|
|
2384
|
-
"HF_HUB_OFFLINE",
|
|
2385
|
-
"HF_HUB_DISABLE_TELEMETRY",
|
|
2386
|
-
"TRANSFORMERS_OFFLINE",
|
|
2387
|
-
"HF_HUB_ENABLE_HF_TRANSFER",
|
|
2388
|
-
"PYTHONUTF8",
|
|
2389
|
-
"PYTHONIOENCODING"
|
|
2390
|
-
]);
|
|
2391
|
-
const prefixes = [
|
|
2392
|
-
"LC_",
|
|
2393
|
-
"CUDA_",
|
|
2394
|
-
"NVIDIA_",
|
|
2395
|
-
"NCCL_",
|
|
2396
|
-
"OMP_",
|
|
2397
|
-
"MKL_",
|
|
2398
|
-
"TORCH_",
|
|
2399
|
-
"PYTORCH_",
|
|
2400
|
-
"TOKENIZERS_",
|
|
2401
|
-
"HF_XET_"
|
|
2402
|
-
];
|
|
2403
|
-
const result = Object.fromEntries(Object.entries(env).filter(
|
|
2404
|
-
([key]) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix))
|
|
2405
|
-
));
|
|
2406
|
-
result.TORCH_DISABLE_NATIVE_JIT ??= "1";
|
|
2407
|
-
return result;
|
|
3007
|
+
function deterministicBehaviorSpecId(spec) {
|
|
3008
|
+
const namespace = Buffer.from("9f783f01340457bea061942004259253", "hex");
|
|
3009
|
+
const digest = createHash3("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
|
|
3010
|
+
digest[6] = digest[6] & 15 | 80;
|
|
3011
|
+
digest[8] = digest[8] & 63 | 128;
|
|
3012
|
+
const hex = digest.toString("hex");
|
|
3013
|
+
return [
|
|
3014
|
+
hex.slice(0, 8),
|
|
3015
|
+
hex.slice(8, 12),
|
|
3016
|
+
hex.slice(12, 16),
|
|
3017
|
+
hex.slice(16, 20),
|
|
3018
|
+
hex.slice(20)
|
|
3019
|
+
].join("-");
|
|
2408
3020
|
}
|
|
2409
|
-
function
|
|
2410
|
-
const
|
|
3021
|
+
async function loadLocalRunInput(path, options = {}) {
|
|
3022
|
+
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile4(path, "utf8")), path);
|
|
3023
|
+
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
3024
|
+
if (request.success) {
|
|
3025
|
+
return { kind: "request", path, request: request.data };
|
|
3026
|
+
}
|
|
3027
|
+
const spec = localBehaviorSpecFileSchema.parse(raw);
|
|
3028
|
+
if (isFoundationSpecFile(spec)) {
|
|
3029
|
+
return { kind: "foundation-spec", path, spec };
|
|
3030
|
+
}
|
|
2411
3031
|
return {
|
|
2412
|
-
|
|
2413
|
-
|
|
3032
|
+
kind: "spec",
|
|
3033
|
+
path,
|
|
3034
|
+
spec,
|
|
3035
|
+
request: runRequestFromLocalSpec(spec, options)
|
|
2414
3036
|
};
|
|
2415
3037
|
}
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
const
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
3038
|
+
|
|
3039
|
+
// src/local-runtime/dataset.ts
|
|
3040
|
+
import { readFile as readFile5 } from "fs/promises";
|
|
3041
|
+
async function loadNormalizedChatJsonl(path) {
|
|
3042
|
+
const text = await readFile5(path, "utf8");
|
|
3043
|
+
const rows = [];
|
|
3044
|
+
const jsonLines = [];
|
|
3045
|
+
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
3046
|
+
if (!line.trim()) continue;
|
|
3047
|
+
let value;
|
|
3048
|
+
try {
|
|
3049
|
+
value = JSON.parse(line);
|
|
3050
|
+
} catch (error) {
|
|
3051
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: malformed JSON`, { cause: error });
|
|
3052
|
+
}
|
|
3053
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
3054
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: expected an object`);
|
|
3055
|
+
}
|
|
3056
|
+
const messages = value.messages;
|
|
3057
|
+
if (!Array.isArray(messages) || ![2, 3].includes(messages.length) || messages.some(
|
|
3058
|
+
(message) => !message || typeof message !== "object" || Array.isArray(message) || !["system", "user", "assistant"].includes(String(message.role)) || typeof message.content !== "string"
|
|
3059
|
+
)) {
|
|
3060
|
+
throw new Error(
|
|
3061
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one user message, and one assistant answer`
|
|
3062
|
+
);
|
|
3063
|
+
}
|
|
3064
|
+
const row = { messages };
|
|
3065
|
+
const offset = row.messages.length === 3 ? 1 : 0;
|
|
3066
|
+
if (offset === 1 && row.messages[0]?.role !== "system" || row.messages[offset]?.role !== "user" || !row.messages[offset]?.content.trim() || row.messages[offset + 1]?.role !== "assistant" || !row.messages[offset + 1]?.content.trim()) {
|
|
3067
|
+
throw new Error(
|
|
3068
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one non-empty user message, and one non-empty assistant answer`
|
|
3069
|
+
);
|
|
3070
|
+
}
|
|
3071
|
+
rows.push(row);
|
|
3072
|
+
jsonLines.push(JSON.stringify(row));
|
|
3073
|
+
}
|
|
3074
|
+
if (rows.length === 0) throw new Error(`Chat JSONL contains no examples: ${path}`);
|
|
3075
|
+
return {
|
|
3076
|
+
rows,
|
|
3077
|
+
jsonl: jsonLines.join("\n")
|
|
2425
3078
|
};
|
|
2426
|
-
delete result.TRANSFORMERS_CACHE;
|
|
2427
|
-
delete result.PYTORCH_TRANSFORMERS_CACHE;
|
|
2428
|
-
delete result.PYTORCH_PRETRAINED_BERT_CACHE;
|
|
2429
|
-
return result;
|
|
2430
3079
|
}
|
|
2431
|
-
function
|
|
3080
|
+
async function normalizeChatJsonlForRelocation(path) {
|
|
3081
|
+
return (await loadNormalizedChatJsonl(path)).jsonl;
|
|
3082
|
+
}
|
|
3083
|
+
function buildSystemMessage(spec) {
|
|
3084
|
+
const parts = [];
|
|
3085
|
+
if (spec.system_prompt.trim()) parts.push(spec.system_prompt.trim());
|
|
3086
|
+
if (spec.guidelines.length > 0) {
|
|
3087
|
+
parts.push(`Guidelines:
|
|
3088
|
+
${spec.guidelines.map((guideline) => `- ${guideline}`).join("\n")}`);
|
|
3089
|
+
}
|
|
3090
|
+
if (spec.constraints.length > 0) {
|
|
3091
|
+
parts.push(`Constraints:
|
|
3092
|
+
${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
|
|
3093
|
+
}
|
|
3094
|
+
return parts.join("\n\n") || "Follow the demonstrated behavior.";
|
|
3095
|
+
}
|
|
3096
|
+
function exampleToChatRow(spec, example) {
|
|
3097
|
+
const system = buildSystemMessage(spec);
|
|
2432
3098
|
return {
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
3099
|
+
messages: [
|
|
3100
|
+
{ role: "system", content: system },
|
|
3101
|
+
{ role: "user", content: example.input },
|
|
3102
|
+
{ role: "assistant", content: example.output }
|
|
3103
|
+
]
|
|
3104
|
+
};
|
|
3105
|
+
}
|
|
3106
|
+
function compileSpecToJsonl(spec) {
|
|
3107
|
+
return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
|
|
3108
|
+
}
|
|
3109
|
+
function examplesFromSpec(spec) {
|
|
3110
|
+
return spec.examples;
|
|
3111
|
+
}
|
|
3112
|
+
async function examplesFromChatJsonl(path) {
|
|
3113
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
3114
|
+
const examples = [];
|
|
3115
|
+
for (const [index, row] of normalized.rows.entries()) {
|
|
3116
|
+
const user = row.messages.at(-2);
|
|
3117
|
+
const assistant = row.messages.at(-1);
|
|
3118
|
+
examples.push({
|
|
3119
|
+
input: user.content,
|
|
3120
|
+
output: assistant.content
|
|
3121
|
+
});
|
|
3122
|
+
}
|
|
3123
|
+
return examples;
|
|
3124
|
+
}
|
|
3125
|
+
async function evaluationSuiteFromChatJsonl(path, configuredSystem) {
|
|
3126
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
3127
|
+
const inputIdentities = /* @__PURE__ */ new Set();
|
|
3128
|
+
for (const row of normalized.rows) {
|
|
3129
|
+
const input = row.messages.at(-2).content;
|
|
3130
|
+
const identity = input.trim().replace(/\s+/g, " ").toLowerCase();
|
|
3131
|
+
if (inputIdentities.has(identity)) {
|
|
3132
|
+
throw new Error(
|
|
3133
|
+
"General regression data contains duplicate inputs; evaluation prompts must be unique."
|
|
3134
|
+
);
|
|
3135
|
+
}
|
|
3136
|
+
inputIdentities.add(identity);
|
|
3137
|
+
}
|
|
3138
|
+
const systems = new Set(
|
|
3139
|
+
normalized.rows.map((row) => row.messages.find((message) => message.role === "system")?.content).filter((value) => value !== void 0)
|
|
3140
|
+
);
|
|
3141
|
+
if (systems.size > 1) {
|
|
3142
|
+
throw new Error("General regression rows must use one consistent system message.");
|
|
3143
|
+
}
|
|
3144
|
+
const embeddedSystem = [...systems][0];
|
|
3145
|
+
if (configuredSystem !== void 0 && embeddedSystem !== void 0 && configuredSystem !== embeddedSystem) {
|
|
3146
|
+
throw new Error(
|
|
3147
|
+
"evaluation.generalRegression.systemPrompt must match the system message embedded in the dataset."
|
|
3148
|
+
);
|
|
3149
|
+
}
|
|
3150
|
+
return {
|
|
3151
|
+
examples: normalized.rows.map((row) => ({
|
|
3152
|
+
input: row.messages.at(-2).content,
|
|
3153
|
+
output: row.messages.at(-1).content
|
|
3154
|
+
})),
|
|
3155
|
+
system: configuredSystem ?? embeddedSystem ?? "You are a helpful assistant."
|
|
2436
3156
|
};
|
|
2437
3157
|
}
|
|
2438
3158
|
|
|
2439
3159
|
// src/local-runtime/prefetch.ts
|
|
3160
|
+
import { createHash as createHash4 } from "crypto";
|
|
3161
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
3162
|
+
import { lstat as lstat4, mkdir as mkdir6, readFile as readFile6, readdir as readdir4, realpath as realpath2, stat as stat4 } from "fs/promises";
|
|
3163
|
+
import { isAbsolute as isAbsolute4, join as join8, relative as relative3, resolve as resolve10, sep as sep4 } from "path";
|
|
2440
3164
|
var NEMOTRON_BF16_FILE_DIGESTS = {
|
|
2441
3165
|
".gitattributes": { algorithm: "sha1", digest: "fe3f694ce14b577c9833fca6e83c5e26a7119127" },
|
|
2442
3166
|
"LICENSE": { algorithm: "sha1", digest: "4d76cc87c9edb280753e3bee3b499ae461113835" },
|
|
@@ -2502,14 +3226,14 @@ async function certifiedFileDigest(path, expected) {
|
|
|
2502
3226
|
return digest.digest("hex");
|
|
2503
3227
|
}
|
|
2504
3228
|
function isStrictPhysicalDescendant(parent, child) {
|
|
2505
|
-
const candidate =
|
|
2506
|
-
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${
|
|
3229
|
+
const candidate = relative3(parent, child);
|
|
3230
|
+
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep4}`) && !isAbsolute4(candidate);
|
|
2507
3231
|
}
|
|
2508
3232
|
async function verifySnapshotCacheLocation(path, modelId, revision, modelCache) {
|
|
2509
3233
|
const repository = `models--${modelId.replaceAll("/", "--")}`;
|
|
2510
3234
|
const layout = resolveHuggingFaceCacheLayout(modelCache);
|
|
2511
|
-
const repositoryRoot =
|
|
2512
|
-
const expectedPath =
|
|
3235
|
+
const repositoryRoot = join8(layout.hubCache, repository);
|
|
3236
|
+
const expectedPath = join8(repositoryRoot, "snapshots", revision);
|
|
2513
3237
|
const [
|
|
2514
3238
|
physicalCacheRoot,
|
|
2515
3239
|
physicalHubRoot,
|
|
@@ -2521,7 +3245,7 @@ async function verifySnapshotCacheLocation(path, modelId, revision, modelCache)
|
|
|
2521
3245
|
realpath2(layout.hfHome).catch(() => null),
|
|
2522
3246
|
realpath2(layout.hubCache).catch(() => null),
|
|
2523
3247
|
realpath2(repositoryRoot).catch(() => null),
|
|
2524
|
-
realpath2(
|
|
3248
|
+
realpath2(join8(repositoryRoot, "blobs")).catch(() => null),
|
|
2525
3249
|
realpath2(path).catch(() => null),
|
|
2526
3250
|
realpath2(expectedPath).catch(() => null)
|
|
2527
3251
|
]);
|
|
@@ -2537,7 +3261,7 @@ async function verifySnapshotCacheLocation(path, modelId, revision, modelCache)
|
|
|
2537
3261
|
throw new Error(`Hugging Face repository escapes its configured hub cache: ${repositoryRoot}`);
|
|
2538
3262
|
}
|
|
2539
3263
|
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalBlobRoot)) {
|
|
2540
|
-
throw new Error(`Hugging Face blob store escapes its repository: ${
|
|
3264
|
+
throw new Error(`Hugging Face blob store escapes its repository: ${join8(repositoryRoot, "blobs")}`);
|
|
2541
3265
|
}
|
|
2542
3266
|
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalExpectedPath)) {
|
|
2543
3267
|
throw new Error(`Hugging Face snapshot escapes its configured repository: ${expectedPath}`);
|
|
@@ -2595,17 +3319,17 @@ async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelC
|
|
|
2595
3319
|
}
|
|
2596
3320
|
physicalFiles.set(file, target);
|
|
2597
3321
|
}
|
|
2598
|
-
const actualFiles = new Set(snapshotFiles.map((file) =>
|
|
3322
|
+
const actualFiles = new Set(snapshotFiles.map((file) => relative3(path, file).split("\\").join("/")));
|
|
2599
3323
|
const expectedFiles = new Set(Object.keys(digests));
|
|
2600
3324
|
const unexpected = [...actualFiles].filter((name) => !expectedFiles.has(name)).sort();
|
|
2601
3325
|
if (unexpected.length > 0) {
|
|
2602
|
-
throw new Error(`Certified snapshot contains unexpected file: ${
|
|
3326
|
+
throw new Error(`Certified snapshot contains unexpected file: ${join8(path, unexpected[0])}`);
|
|
2603
3327
|
}
|
|
2604
3328
|
for (const [name, expected] of Object.entries(digests)) {
|
|
2605
3329
|
if (!actualFiles.has(name)) {
|
|
2606
|
-
throw new Error(`Certified snapshot file is missing: ${
|
|
3330
|
+
throw new Error(`Certified snapshot file is missing: ${join8(path, name)}`);
|
|
2607
3331
|
}
|
|
2608
|
-
const file =
|
|
3332
|
+
const file = join8(path, name);
|
|
2609
3333
|
const physicalFile = physicalFiles.get(file);
|
|
2610
3334
|
if (await certifiedFileDigest(physicalFile, expected) !== expected.digest) {
|
|
2611
3335
|
throw new Error(`Certified snapshot file checksum mismatch: ${file}`);
|
|
@@ -2624,7 +3348,7 @@ function buildModelPrefetchPayload(request, config) {
|
|
|
2624
3348
|
return {
|
|
2625
3349
|
base_model: model.id,
|
|
2626
3350
|
...revision ? { revision } : {},
|
|
2627
|
-
...config.paths.modelCache ? { model_cache:
|
|
3351
|
+
...config.paths.modelCache ? { model_cache: resolve10(config.paths.modelCache) } : {}
|
|
2628
3352
|
};
|
|
2629
3353
|
}
|
|
2630
3354
|
function safeName(value) {
|
|
@@ -2648,7 +3372,7 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2648
3372
|
const files = [];
|
|
2649
3373
|
const visit = async (directory) => {
|
|
2650
3374
|
for (const entry of await readdir4(directory, { withFileTypes: true })) {
|
|
2651
|
-
const child =
|
|
3375
|
+
const child = join8(directory, entry.name);
|
|
2652
3376
|
if (entry.isSymbolicLink()) {
|
|
2653
3377
|
const metadata2 = await stat4(child);
|
|
2654
3378
|
if (!metadata2.isFile()) {
|
|
@@ -2672,15 +3396,15 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2672
3396
|
throw new Error(`Local base-model directory contains no non-empty Transformers model weights: ${path}`);
|
|
2673
3397
|
}
|
|
2674
3398
|
const required = async (name) => {
|
|
2675
|
-
const metadata = await stat4(
|
|
3399
|
+
const metadata = await stat4(join8(path, name)).catch(() => null);
|
|
2676
3400
|
return Boolean(metadata?.isFile() && metadata.size > 0);
|
|
2677
3401
|
};
|
|
2678
3402
|
if (!await required("config.json")) throw new Error(`Local base-model directory is missing config.json: ${path}`);
|
|
2679
3403
|
try {
|
|
2680
|
-
const config = JSON.parse(await
|
|
3404
|
+
const config = JSON.parse(await readFile6(join8(path, "config.json"), "utf8"));
|
|
2681
3405
|
assertCertifiedBaseModelConfig(
|
|
2682
3406
|
config,
|
|
2683
|
-
`Local base-model config ${
|
|
3407
|
+
`Local base-model config ${join8(path, "config.json")}`,
|
|
2684
3408
|
expectedModelId
|
|
2685
3409
|
);
|
|
2686
3410
|
} catch (error) {
|
|
@@ -2710,7 +3434,7 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2710
3434
|
throw new Error(`Local base-model directory is missing tokenizer metadata or vocabulary: ${path}`);
|
|
2711
3435
|
}
|
|
2712
3436
|
for (const file of files.filter((candidate) => candidate.path.endsWith(".index.json"))) {
|
|
2713
|
-
const parsed = JSON.parse(await
|
|
3437
|
+
const parsed = JSON.parse(await readFile6(file.path, "utf8"));
|
|
2714
3438
|
if (!parsed.weight_map || typeof parsed.weight_map !== "object" || Array.isArray(parsed.weight_map)) {
|
|
2715
3439
|
throw new Error(`Local base-model weight index is invalid: ${file.path}`);
|
|
2716
3440
|
}
|
|
@@ -2730,14 +3454,14 @@ async function prefetchBaseModel(args) {
|
|
|
2730
3454
|
...buildModelPrefetchPayload(args.request, args.config),
|
|
2731
3455
|
...args.localOnly ? { local_files_only: true } : {}
|
|
2732
3456
|
};
|
|
2733
|
-
const artifactDir =
|
|
3457
|
+
const artifactDir = resolve10(
|
|
2734
3458
|
args.config.artifactRoot,
|
|
2735
3459
|
args.localOnly ? "verify-base" : "prefetch",
|
|
2736
3460
|
`${safeName(args.request.spec_snapshot.base_model)}-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
|
|
2737
3461
|
);
|
|
2738
|
-
await
|
|
3462
|
+
await mkdir6(artifactDir, { recursive: true });
|
|
2739
3463
|
if (args.config.paths.baseModel) {
|
|
2740
|
-
const localPath =
|
|
3464
|
+
const localPath = resolve10(args.config.paths.baseModel);
|
|
2741
3465
|
const verified = await verifyLocalBaseModel(
|
|
2742
3466
|
localPath,
|
|
2743
3467
|
args.request.spec_snapshot.base_model,
|
|
@@ -2754,9 +3478,9 @@ async function prefetchBaseModel(args) {
|
|
|
2754
3478
|
reason: args.localOnly ? "Verified the configured local base-model artifact; no network access was used." : "paths.baseModel points at a verified local base-model artifact; no Hugging Face download is needed."
|
|
2755
3479
|
};
|
|
2756
3480
|
}
|
|
2757
|
-
const inputPath =
|
|
2758
|
-
const outputPath =
|
|
2759
|
-
const logPath =
|
|
3481
|
+
const inputPath = join8(artifactDir, "prefetch-input.json");
|
|
3482
|
+
const outputPath = join8(artifactDir, "prefetch-output.json");
|
|
3483
|
+
const logPath = join8(artifactDir, "prefetch.log");
|
|
2760
3484
|
await writeJson(inputPath, payload);
|
|
2761
3485
|
const entrypoint = buildBundledPythonCommand(
|
|
2762
3486
|
"prefetch.py",
|
|
@@ -2793,7 +3517,7 @@ async function prefetchBaseModel(args) {
|
|
|
2793
3517
|
if (exitCode !== 0) {
|
|
2794
3518
|
throw new Error(`Model prefetch exited with code ${exitCode}. See ${logPath}.`);
|
|
2795
3519
|
}
|
|
2796
|
-
const output = JSON.parse(await
|
|
3520
|
+
const output = JSON.parse(await readFile6(outputPath, "utf8"));
|
|
2797
3521
|
const snapshotRevision = baseModelRevisionSchema.safeParse(output.snapshot_revision);
|
|
2798
3522
|
if (!snapshotRevision.success) {
|
|
2799
3523
|
throw new Error(
|
|
@@ -2876,17 +3600,17 @@ function evaluateGeneralRegressionGate(comparison, policy) {
|
|
|
2876
3600
|
}
|
|
2877
3601
|
|
|
2878
3602
|
// src/local-runtime/orchestrator.ts
|
|
2879
|
-
import { lstat as lstat5, mkdir as
|
|
3603
|
+
import { lstat as lstat5, mkdir as mkdir9, open as open3, readFile as readFile9, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
|
|
2880
3604
|
import { createReadStream as createReadStream3 } from "fs";
|
|
2881
3605
|
import { createHash as createHash6, randomUUID as randomUUID4 } from "crypto";
|
|
2882
3606
|
import { performance } from "perf_hooks";
|
|
2883
|
-
import { homedir as
|
|
2884
|
-
import { basename, dirname as
|
|
3607
|
+
import { homedir as homedir5 } from "os";
|
|
3608
|
+
import { basename, dirname as dirname10, join as join11, relative as relative4, resolve as resolve13 } from "path";
|
|
2885
3609
|
|
|
2886
3610
|
// src/local-runtime/evaluation.ts
|
|
2887
3611
|
import { createHash as createHash5 } from "crypto";
|
|
2888
|
-
import { mkdir as
|
|
2889
|
-
import { dirname as
|
|
3612
|
+
import { mkdir as mkdir7, readFile as readFile7, rm as rm2, writeFile as writeFile5 } from "fs/promises";
|
|
3613
|
+
import { dirname as dirname9, join as join9, resolve as resolve11 } from "path";
|
|
2890
3614
|
function normalize(value) {
|
|
2891
3615
|
return value.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2892
3616
|
}
|
|
@@ -3106,8 +3830,8 @@ async function runTransformersInference(args) {
|
|
|
3106
3830
|
const inputPath = `${args.outputPath}.inference-input.json`;
|
|
3107
3831
|
const outputPath = `${args.outputPath}.inference-output.json`;
|
|
3108
3832
|
const logPath = `${args.outputPath}.inference.log`;
|
|
3109
|
-
await
|
|
3110
|
-
await
|
|
3833
|
+
await mkdir7(dirname9(inputPath), { recursive: true });
|
|
3834
|
+
await writeFile5(inputPath, `${JSON.stringify({
|
|
3111
3835
|
protocol_version: INFERENCE_PROTOCOL_VERSION,
|
|
3112
3836
|
kind: args.kind,
|
|
3113
3837
|
model_id: args.modelId,
|
|
@@ -3121,7 +3845,7 @@ async function runTransformersInference(args) {
|
|
|
3121
3845
|
id: String(index),
|
|
3122
3846
|
input: example.input
|
|
3123
3847
|
})),
|
|
3124
|
-
model_cache: args.config.paths.modelCache ?
|
|
3848
|
+
model_cache: args.config.paths.modelCache ? resolve11(args.config.paths.modelCache) : void 0,
|
|
3125
3849
|
trust_remote_code: false,
|
|
3126
3850
|
device: args.config.evaluation.inference.device,
|
|
3127
3851
|
generation: {
|
|
@@ -3170,7 +3894,7 @@ async function runTransformersInference(args) {
|
|
|
3170
3894
|
}
|
|
3171
3895
|
let output;
|
|
3172
3896
|
try {
|
|
3173
|
-
output = JSON.parse(await
|
|
3897
|
+
output = JSON.parse(await readFile7(outputPath, "utf8"));
|
|
3174
3898
|
} catch (error) {
|
|
3175
3899
|
throw new Error(
|
|
3176
3900
|
`Transformers inference did not write valid JSON: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -3207,15 +3931,15 @@ function baselineInputsAreStable(args) {
|
|
|
3207
3931
|
return args.config.paths.baseModel ? Boolean(args.sourceFingerprint) : Boolean(args.baseModelRevision);
|
|
3208
3932
|
}
|
|
3209
3933
|
function baselineStoreRoot(config) {
|
|
3210
|
-
return config.storeRoot ?
|
|
3934
|
+
return config.storeRoot ? resolve11(config.storeRoot) : defaultLocalHome();
|
|
3211
3935
|
}
|
|
3212
3936
|
function baselineCachePath(config, key) {
|
|
3213
|
-
return
|
|
3937
|
+
return join9(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
|
|
3214
3938
|
}
|
|
3215
3939
|
async function readBaselineCache(config, path) {
|
|
3216
3940
|
await hardenExistingLocalStore(baselineStoreRoot(config));
|
|
3217
3941
|
try {
|
|
3218
|
-
return evalReportSchema.parse(JSON.parse(await
|
|
3942
|
+
return evalReportSchema.parse(JSON.parse(await readFile7(path, "utf8")));
|
|
3219
3943
|
} catch {
|
|
3220
3944
|
return null;
|
|
3221
3945
|
}
|
|
@@ -3224,7 +3948,7 @@ var cachedPackageVersion = null;
|
|
|
3224
3948
|
async function packageVersion() {
|
|
3225
3949
|
if (cachedPackageVersion !== null) return cachedPackageVersion;
|
|
3226
3950
|
try {
|
|
3227
|
-
const raw = await
|
|
3951
|
+
const raw = await readFile7(resolve11(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
|
|
3228
3952
|
cachedPackageVersion = String(
|
|
3229
3953
|
JSON.parse(raw).version ?? "unknown"
|
|
3230
3954
|
);
|
|
@@ -3293,7 +4017,7 @@ async function evaluateExamples(args) {
|
|
|
3293
4017
|
baseModelId,
|
|
3294
4018
|
args.baseModelRevision
|
|
3295
4019
|
);
|
|
3296
|
-
const localModelSource = args.modelSource ?
|
|
4020
|
+
const localModelSource = args.modelSource ? resolve11(args.modelSource) !== resolve11(baseModelId) : false;
|
|
3297
4021
|
const baseModelRevision = localModelSource ? defaultBaseModelRevision(baseModelId) : requestedBaseModelRevision;
|
|
3298
4022
|
const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
|
|
3299
4023
|
const truncated = args.examples.length > maxExamples;
|
|
@@ -3409,8 +4133,8 @@ function compareEvalReports(baseline, candidate) {
|
|
|
3409
4133
|
}
|
|
3410
4134
|
|
|
3411
4135
|
// src/local-runtime/process-training.ts
|
|
3412
|
-
import { access, mkdir as
|
|
3413
|
-
import { join as
|
|
4136
|
+
import { access as access2, mkdir as mkdir8, readFile as readFile8, writeFile as writeFile6 } from "fs/promises";
|
|
4137
|
+
import { join as join10, resolve as resolve12 } from "path";
|
|
3414
4138
|
function buildTrainingHyperparameters(request, options = {}) {
|
|
3415
4139
|
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
3416
4140
|
const hyper = request.hyperparameters;
|
|
@@ -3434,7 +4158,7 @@ function buildTrainingHyperparameters(request, options = {}) {
|
|
|
3434
4158
|
}
|
|
3435
4159
|
async function pathExists(path) {
|
|
3436
4160
|
try {
|
|
3437
|
-
await
|
|
4161
|
+
await access2(path);
|
|
3438
4162
|
return true;
|
|
3439
4163
|
} catch {
|
|
3440
4164
|
return false;
|
|
@@ -3501,15 +4225,15 @@ function createTrainingProgressForwarder(reporter) {
|
|
|
3501
4225
|
async function launchProcessTraining(args) {
|
|
3502
4226
|
const { request, artifacts, config } = args;
|
|
3503
4227
|
const jobName = `tt-local-${request.run_id}`;
|
|
3504
|
-
await
|
|
4228
|
+
await mkdir8(artifacts.trainingDir, { recursive: true });
|
|
3505
4229
|
await copyDatasetToTrainingChannel(artifacts);
|
|
3506
4230
|
const hyperparameters = buildTrainingHyperparameters(request, {
|
|
3507
4231
|
baseModelRevision: args.baseModelRevision
|
|
3508
4232
|
});
|
|
3509
|
-
await writeJson(
|
|
4233
|
+
await writeJson(join10(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
|
|
3510
4234
|
const entrypoint = buildBundledPythonCommand("train.py");
|
|
3511
4235
|
if (config.dryRun) {
|
|
3512
|
-
await
|
|
4236
|
+
await writeFile6(
|
|
3513
4237
|
artifacts.trainingLog,
|
|
3514
4238
|
"Dry run enabled. Training process was not launched.\n",
|
|
3515
4239
|
"utf8"
|
|
@@ -3531,20 +4255,20 @@ async function launchProcessTraining(args) {
|
|
|
3531
4255
|
withOfflineHuggingFaceCacheEnvironment({
|
|
3532
4256
|
...inheritedEnv,
|
|
3533
4257
|
BACKEND: "local",
|
|
3534
|
-
SM_CHANNEL_TRAINING:
|
|
3535
|
-
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ?
|
|
3536
|
-
SM_MODEL_DIR:
|
|
3537
|
-
SM_OUTPUT_DIR:
|
|
3538
|
-
TT_HYPERPARAMETERS_PATH:
|
|
4258
|
+
SM_CHANNEL_TRAINING: resolve12(artifacts.trainingInputDir),
|
|
4259
|
+
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve12(config.paths.baseModel) : void 0,
|
|
4260
|
+
SM_MODEL_DIR: resolve12(artifacts.trainingModelDir),
|
|
4261
|
+
SM_OUTPUT_DIR: resolve12(artifacts.trainingOutputDir),
|
|
4262
|
+
TT_HYPERPARAMETERS_PATH: resolve12(artifacts.trainingConfigDir, "hyperparameters.json")
|
|
3539
4263
|
}, config.paths.modelCache)
|
|
3540
4264
|
);
|
|
3541
4265
|
for (const [key, value] of Object.entries(childEnv)) {
|
|
3542
4266
|
if (value === void 0) delete childEnv[key];
|
|
3543
4267
|
}
|
|
3544
|
-
if (config.paths.modelCache) await
|
|
4268
|
+
if (config.paths.modelCache) await mkdir8(resolve12(config.paths.modelCache), { recursive: true });
|
|
3545
4269
|
if (config.paths.baseModel) {
|
|
3546
4270
|
await verifyLocalBaseModel(
|
|
3547
|
-
|
|
4271
|
+
resolve12(config.paths.baseModel),
|
|
3548
4272
|
request.spec_snapshot.base_model,
|
|
3549
4273
|
config.paths.modelCache
|
|
3550
4274
|
);
|
|
@@ -3571,12 +4295,12 @@ async function launchProcessTraining(args) {
|
|
|
3571
4295
|
onLine: (line) => forwardTrainingProgress(line),
|
|
3572
4296
|
shouldCancel: args.shouldCancel
|
|
3573
4297
|
});
|
|
3574
|
-
const metricsPath =
|
|
3575
|
-
const metrics = await pathExists(metricsPath) ? JSON.parse(await
|
|
4298
|
+
const metricsPath = join10(artifacts.trainingModelDir, "training-metrics.json");
|
|
4299
|
+
const metrics = await pathExists(metricsPath) ? JSON.parse(await readFile8(metricsPath, "utf8")) : null;
|
|
3576
4300
|
if (exitCode !== 0) {
|
|
3577
4301
|
throw new Error(`Training process exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
|
|
3578
4302
|
}
|
|
3579
|
-
const modelTarPath =
|
|
4303
|
+
const modelTarPath = join10(artifacts.trainingOutputDir, "model.tar.gz");
|
|
3580
4304
|
const modelUri = await pathExists(modelTarPath) ? fileUri(modelTarPath) : fileUri(artifacts.trainingModelDir);
|
|
3581
4305
|
await args.reporter?.onEvent?.({
|
|
3582
4306
|
stage: "training",
|
|
@@ -3671,21 +4395,21 @@ function validateLocalPipeline(pipeline) {
|
|
|
3671
4395
|
return pipeline;
|
|
3672
4396
|
}
|
|
3673
4397
|
async function loadJsonFile(path) {
|
|
3674
|
-
return JSON.parse(await
|
|
4398
|
+
return JSON.parse(await readFile9(path, "utf8"));
|
|
3675
4399
|
}
|
|
3676
4400
|
async function loadRunRequest(path) {
|
|
3677
4401
|
return fineTuneRunRequestSchema.parse(await loadJsonFile(path));
|
|
3678
4402
|
}
|
|
3679
4403
|
async function loadLocalRunnerConfig(path) {
|
|
3680
4404
|
if (!path) return localRunnerConfigSchema.parse({});
|
|
3681
|
-
const configPath =
|
|
3682
|
-
const base =
|
|
4405
|
+
const configPath = resolve13(path);
|
|
4406
|
+
const base = dirname10(configPath);
|
|
3683
4407
|
const config = localRunnerConfigSchema.parse(await loadJsonFile(configPath));
|
|
3684
4408
|
const configPathValue = (value) => {
|
|
3685
4409
|
if (!value) return void 0;
|
|
3686
|
-
if (value === "~") return
|
|
3687
|
-
if (value.startsWith("~/")) return
|
|
3688
|
-
return
|
|
4410
|
+
if (value === "~") return homedir5();
|
|
4411
|
+
if (value.startsWith("~/")) return resolve13(homedir5(), value.slice(2));
|
|
4412
|
+
return resolve13(base, value);
|
|
3689
4413
|
};
|
|
3690
4414
|
return {
|
|
3691
4415
|
...config,
|
|
@@ -3883,21 +4607,21 @@ async function hashFileIfPresent(path) {
|
|
|
3883
4607
|
}
|
|
3884
4608
|
async function packageVersion2() {
|
|
3885
4609
|
try {
|
|
3886
|
-
const parsed = JSON.parse(await
|
|
4610
|
+
const parsed = JSON.parse(await readFile9(resolve13(packageRoot2, "package.json"), "utf8"));
|
|
3887
4611
|
return typeof parsed.version === "string" ? parsed.version : "unknown";
|
|
3888
4612
|
} catch {
|
|
3889
4613
|
return "unknown";
|
|
3890
4614
|
}
|
|
3891
4615
|
}
|
|
3892
4616
|
async function runtimeFingerprint() {
|
|
3893
|
-
const bundledProject2 =
|
|
4617
|
+
const bundledProject2 = resolve13(packageRoot2, "training/adapter");
|
|
3894
4618
|
return hashJson({
|
|
3895
4619
|
tt_local_version: await packageVersion2(),
|
|
3896
4620
|
node_version: process.version,
|
|
3897
4621
|
platform: process.platform,
|
|
3898
4622
|
architecture: process.arch,
|
|
3899
|
-
runner_pyproject: await hashFileIfPresent(
|
|
3900
|
-
runner_uv_lock: await hashFileIfPresent(
|
|
4623
|
+
runner_pyproject: await hashFileIfPresent(resolve13(bundledProject2, "pyproject.toml")),
|
|
4624
|
+
runner_uv_lock: await hashFileIfPresent(resolve13(bundledProject2, "uv.lock"))
|
|
3901
4625
|
});
|
|
3902
4626
|
}
|
|
3903
4627
|
function preparedSourceFingerprint(args) {
|
|
@@ -3944,9 +4668,9 @@ async function resolveBaseModelRevision(request, config) {
|
|
|
3944
4668
|
}
|
|
3945
4669
|
const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
|
|
3946
4670
|
const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
|
|
3947
|
-
const refPath =
|
|
4671
|
+
const refPath = resolve13(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
|
|
3948
4672
|
try {
|
|
3949
|
-
const revision = (await
|
|
4673
|
+
const revision = (await readFile9(refPath, "utf8")).trim();
|
|
3950
4674
|
if (revision) return revision;
|
|
3951
4675
|
} catch {
|
|
3952
4676
|
}
|
|
@@ -3965,7 +4689,7 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
|
|
|
3965
4689
|
const files = [];
|
|
3966
4690
|
const visit = async (directory) => {
|
|
3967
4691
|
for (const entry of await readdir5(directory, { withFileTypes: true })) {
|
|
3968
|
-
const child =
|
|
4692
|
+
const child = resolve13(directory, entry.name);
|
|
3969
4693
|
if (entry.isDirectory()) {
|
|
3970
4694
|
await visit(child);
|
|
3971
4695
|
continue;
|
|
@@ -3976,7 +4700,7 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
|
|
|
3976
4700
|
}
|
|
3977
4701
|
if (!childMetadata.isFile()) continue;
|
|
3978
4702
|
files.push({
|
|
3979
|
-
path:
|
|
4703
|
+
path: relative4(root, child).split("\\").join("/"),
|
|
3980
4704
|
size_bytes: childMetadata.size,
|
|
3981
4705
|
sha256: await hashFile(child)
|
|
3982
4706
|
});
|
|
@@ -4004,12 +4728,12 @@ async function readStageMetadata(path) {
|
|
|
4004
4728
|
}
|
|
4005
4729
|
}
|
|
4006
4730
|
async function removePrefixedArtifacts(path) {
|
|
4007
|
-
const directory =
|
|
4731
|
+
const directory = dirname10(path);
|
|
4008
4732
|
const prefix = `${basename(path)}.`;
|
|
4009
4733
|
const names = await readdir5(directory).catch(() => []);
|
|
4010
4734
|
await Promise.all([
|
|
4011
4735
|
rm3(path, { force: true }),
|
|
4012
|
-
...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(
|
|
4736
|
+
...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(resolve13(directory, name), { recursive: true, force: true }))
|
|
4013
4737
|
]);
|
|
4014
4738
|
}
|
|
4015
4739
|
async function cleanupStageArtifacts(artifacts, stage) {
|
|
@@ -4022,7 +4746,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
4022
4746
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
4023
4747
|
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
4024
4748
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
4025
|
-
rm3(
|
|
4749
|
+
rm3(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
4026
4750
|
removeReport(),
|
|
4027
4751
|
rm3(artifacts.artifactManifestJson, { force: true })
|
|
4028
4752
|
]);
|
|
@@ -4040,7 +4764,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
4040
4764
|
await Promise.all([
|
|
4041
4765
|
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
4042
4766
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
4043
|
-
rm3(
|
|
4767
|
+
rm3(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
4044
4768
|
removePrefixedArtifacts(artifacts.candidateEvalJson),
|
|
4045
4769
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
4046
4770
|
removeReport()
|
|
@@ -4059,7 +4783,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
4059
4783
|
await removeReport();
|
|
4060
4784
|
}
|
|
4061
4785
|
function manifestRelativePath(artifacts, path) {
|
|
4062
|
-
return
|
|
4786
|
+
return relative4(artifacts.runDir, path).split("\\").join("/");
|
|
4063
4787
|
}
|
|
4064
4788
|
async function verifyReusableArtifacts(artifacts, paths, options = {}) {
|
|
4065
4789
|
if (!await pathExists2(artifacts.artifactManifestJson)) return false;
|
|
@@ -4076,11 +4800,11 @@ async function throwIfCancelled(store, request) {
|
|
|
4076
4800
|
}
|
|
4077
4801
|
}
|
|
4078
4802
|
async function acquireWorkflowLock(lockPath, description) {
|
|
4079
|
-
await
|
|
4803
|
+
await mkdir9(dirname10(lockPath), { recursive: true });
|
|
4080
4804
|
const token = randomUUID4();
|
|
4081
4805
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4082
4806
|
try {
|
|
4083
|
-
const handle = await
|
|
4807
|
+
const handle = await open3(lockPath, "wx", 384);
|
|
4084
4808
|
try {
|
|
4085
4809
|
await handle.writeFile(`${JSON.stringify({ token, pid: process.pid, created_at: (/* @__PURE__ */ new Date()).toISOString() })}
|
|
4086
4810
|
`);
|
|
@@ -4089,7 +4813,7 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
4089
4813
|
}
|
|
4090
4814
|
return async () => {
|
|
4091
4815
|
try {
|
|
4092
|
-
const owner = JSON.parse(await
|
|
4816
|
+
const owner = JSON.parse(await readFile9(lockPath, "utf8"));
|
|
4093
4817
|
if (owner.token === token) await rm3(lockPath, { force: true });
|
|
4094
4818
|
} catch {
|
|
4095
4819
|
}
|
|
@@ -4099,7 +4823,7 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
4099
4823
|
const metadata = await stat5(lockPath).catch(() => null);
|
|
4100
4824
|
let owner = null;
|
|
4101
4825
|
try {
|
|
4102
|
-
owner = JSON.parse(await
|
|
4826
|
+
owner = JSON.parse(await readFile9(lockPath, "utf8"));
|
|
4103
4827
|
} catch {
|
|
4104
4828
|
owner = null;
|
|
4105
4829
|
}
|
|
@@ -4131,11 +4855,11 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
4131
4855
|
}
|
|
4132
4856
|
async function acquireRunLock(store, runId) {
|
|
4133
4857
|
await store.ensure();
|
|
4134
|
-
return acquireWorkflowLock(
|
|
4858
|
+
return acquireWorkflowLock(resolve13(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
|
|
4135
4859
|
}
|
|
4136
4860
|
async function acquireArtifactLock(artifacts) {
|
|
4137
4861
|
return acquireWorkflowLock(
|
|
4138
|
-
|
|
4862
|
+
join11(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
|
|
4139
4863
|
`Artifact directory ${artifacts.runDir}`
|
|
4140
4864
|
);
|
|
4141
4865
|
}
|
|
@@ -4182,13 +4906,13 @@ function stageFingerprintPath(prepared, stage) {
|
|
|
4182
4906
|
return `${stageOutputPath(prepared, stage)}.stage.json`;
|
|
4183
4907
|
}
|
|
4184
4908
|
async function runnerFileFingerprints() {
|
|
4185
|
-
const project =
|
|
4186
|
-
const bundledSource =
|
|
4187
|
-
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) =>
|
|
4909
|
+
const project = resolve13(packageRoot2, "training/adapter");
|
|
4910
|
+
const bundledSource = resolve13(packageRoot2, "training/adapter/src");
|
|
4911
|
+
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve13(bundledSource, entry.name));
|
|
4188
4912
|
const candidates = [.../* @__PURE__ */ new Set([
|
|
4189
4913
|
...pythonSources,
|
|
4190
|
-
|
|
4191
|
-
|
|
4914
|
+
resolve13(project, "pyproject.toml"),
|
|
4915
|
+
resolve13(project, "uv.lock")
|
|
4192
4916
|
])];
|
|
4193
4917
|
const fingerprints = {};
|
|
4194
4918
|
for (const path of candidates.sort()) fingerprints[path] = await hashFileIfPresent(path);
|
|
@@ -4302,7 +5026,7 @@ async function ensureRunRecord(args) {
|
|
|
4302
5026
|
`Run ${args.request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
|
|
4303
5027
|
);
|
|
4304
5028
|
}
|
|
4305
|
-
if (
|
|
5029
|
+
if (resolve13(existing.artifact_dir) !== resolve13(args.artifacts.runDir)) {
|
|
4306
5030
|
throw new Error(
|
|
4307
5031
|
`Run ${args.request.run_id} is already bound to artifact directory ${existing.artifact_dir}; it cannot be resumed with ${args.artifacts.runDir}.`
|
|
4308
5032
|
);
|
|
@@ -4409,7 +5133,7 @@ async function prepareStage(args) {
|
|
|
4409
5133
|
const preparedExists = await pathExists2(args.artifacts.stageMetadataJson) && await pathExists2(args.artifacts.trainingJsonl);
|
|
4410
5134
|
const prepared = await computePreparedRun({ ...args, writeArtifacts: false });
|
|
4411
5135
|
await args.store.syncRunRequest(args.request, args.artifacts.runDir);
|
|
4412
|
-
await writeJsonAtomic(
|
|
5136
|
+
await writeJsonAtomic(resolve13(args.artifacts.runDir, "request.json"), args.request);
|
|
4413
5137
|
const existingMetadata = preparedExists ? await readStageMetadata(args.artifacts.stageMetadataJson) : null;
|
|
4414
5138
|
let canReuse = preparedExists && existingMetadata?.source_fingerprint === prepared.metadata.source_fingerprint;
|
|
4415
5139
|
if (canReuse) {
|
|
@@ -4938,7 +5662,7 @@ async function runLocalPipeline(input) {
|
|
|
4938
5662
|
if (step.uses === "evaluate") {
|
|
4939
5663
|
const isBase = step.with.model === "base";
|
|
4940
5664
|
const report = isBase ? await runBaselineStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter }) : await runCandidateStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
|
|
4941
|
-
const reportPath =
|
|
5665
|
+
const reportPath = join11(artifacts.runDir, "evaluations", `${step.id}.json`);
|
|
4942
5666
|
await writeJsonAtomic(reportPath, { ...report, artifact_uri: fileUri(reportPath) });
|
|
4943
5667
|
outputs[step.id] = { report: { ...report, artifact_uri: fileUri(reportPath) }, reportPath };
|
|
4944
5668
|
await refreshArtifactManifest(prepared);
|
|
@@ -5027,6 +5751,23 @@ export {
|
|
|
5027
5751
|
getConfigDir,
|
|
5028
5752
|
getAgentConfigDir,
|
|
5029
5753
|
defaultStoreRoot,
|
|
5754
|
+
warningsFromSnapshot,
|
|
5755
|
+
minimalMachineLearningEnvironment,
|
|
5756
|
+
withHuggingFaceCacheEnvironment,
|
|
5757
|
+
withOfflineHuggingFaceCacheEnvironment,
|
|
5758
|
+
reportInBackground,
|
|
5759
|
+
sanitizeLogLine,
|
|
5760
|
+
forwardStreamLines,
|
|
5761
|
+
localRuntimePackageRoot,
|
|
5762
|
+
buildBundledPythonCommand,
|
|
5763
|
+
withBundledPythonEnvironment,
|
|
5764
|
+
buildFoundationPythonCommand,
|
|
5765
|
+
withFoundationPythonEnvironment,
|
|
5766
|
+
runLoggedProcess,
|
|
5767
|
+
readHardwareSnapshot,
|
|
5768
|
+
formatHostStatusLine,
|
|
5769
|
+
formatAgentHostBlock,
|
|
5770
|
+
assessHardware,
|
|
5030
5771
|
isTerminalRunState,
|
|
5031
5772
|
defaultLocalHome,
|
|
5032
5773
|
localStorePaths,
|
|
@@ -5054,6 +5795,7 @@ export {
|
|
|
5054
5795
|
localRunnerConfigSchema,
|
|
5055
5796
|
isFoundationSpecFile,
|
|
5056
5797
|
DEFAULT_LOCAL_SPEC_PATH,
|
|
5798
|
+
localPathsOverlap,
|
|
5057
5799
|
resolveLocalRunInputPaths,
|
|
5058
5800
|
initLocalSpecFile,
|
|
5059
5801
|
initLocalRunnerConfigFile,
|
|
@@ -5070,18 +5812,6 @@ export {
|
|
|
5070
5812
|
examplesFromSpec,
|
|
5071
5813
|
examplesFromChatJsonl,
|
|
5072
5814
|
evaluationSuiteFromChatJsonl,
|
|
5073
|
-
reportInBackground,
|
|
5074
|
-
sanitizeLogLine,
|
|
5075
|
-
forwardStreamLines,
|
|
5076
|
-
localRuntimePackageRoot,
|
|
5077
|
-
buildBundledPythonCommand,
|
|
5078
|
-
withBundledPythonEnvironment,
|
|
5079
|
-
buildFoundationPythonCommand,
|
|
5080
|
-
withFoundationPythonEnvironment,
|
|
5081
|
-
runLoggedProcess,
|
|
5082
|
-
minimalMachineLearningEnvironment,
|
|
5083
|
-
withHuggingFaceCacheEnvironment,
|
|
5084
|
-
withOfflineHuggingFaceCacheEnvironment,
|
|
5085
5815
|
tokenF1,
|
|
5086
5816
|
verifyModelPrefetchCacheReport,
|
|
5087
5817
|
buildModelPrefetchPayload,
|
|
@@ -5098,4 +5828,4 @@ export {
|
|
|
5098
5828
|
runLocalPipeline,
|
|
5099
5829
|
runLocalFineTune
|
|
5100
5830
|
};
|
|
5101
|
-
//# sourceMappingURL=chunk-
|
|
5831
|
+
//# sourceMappingURL=chunk-W7MOBPCO.js.map
|