@tuned-tensor/cli 0.13.1-beta.4 → 0.13.1-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -2
- package/dist/{chunk-7VIUHQBI.js → chunk-AVMRHDFY.js} +1499 -792
- package/dist/chunk-AVMRHDFY.js.map +1 -0
- package/dist/index.js +155 -18
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +31 -1
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/spark.md +6 -0
- package/package.json +1 -1
- package/dist/chunk-7VIUHQBI.js.map +0 -1
|
@@ -477,37 +477,37 @@ async function verifyTarGzipArchive(path) {
|
|
|
477
477
|
if (sizeText && !/^[0-7]+$/.test(sizeText)) throw new Error(`Invalid tar member size in ${path}.`);
|
|
478
478
|
const size = sizeText ? Number.parseInt(sizeText, 8) : 0;
|
|
479
479
|
if (!Number.isSafeInteger(size) || size < 0) throw new Error(`Invalid tar member size in ${path}.`);
|
|
480
|
-
const
|
|
481
|
-
if (
|
|
482
|
-
throw new Error(`Unsafe or unsupported tar member type ${JSON.stringify(
|
|
480
|
+
const type2 = String.fromCharCode(header[156] ?? 0);
|
|
481
|
+
if (type2 !== "\0" && type2 !== "0" && type2 !== "5" && type2 !== "x" && type2 !== "g") {
|
|
482
|
+
throw new Error(`Unsafe or unsupported tar member type ${JSON.stringify(type2)} in ${path}.`);
|
|
483
483
|
}
|
|
484
484
|
const shortName = header.subarray(0, 100).toString("utf8").replace(/\0.*$/, "");
|
|
485
485
|
const prefix = header.subarray(345, 500).toString("utf8").replace(/\0.*$/, "");
|
|
486
486
|
const headerPath = safeMemberPath(prefix ? `${prefix}/${shortName}` : shortName);
|
|
487
|
-
const memberPath =
|
|
488
|
-
if (
|
|
487
|
+
const memberPath = type2 === "x" || type2 === "g" ? headerPath : safeMemberPath(pendingPaxPath ?? headerPath);
|
|
488
|
+
if (type2 !== "x" && type2 !== "g") pendingPaxPath = void 0;
|
|
489
489
|
const segments = memberPath.split("/").filter(Boolean);
|
|
490
490
|
const name = segments.at(-1)?.toLowerCase();
|
|
491
491
|
const metadataOnly = name === "training-metrics.json" || name === "trainer_state.json" || name === "training_args.bin" || name === "config.json" || name === "generation_config.json" || name === "adapter_config.json" || name === "readme.md" || name?.startsWith("tokenizer") || name?.startsWith("special_tokens") || name?.startsWith("added_tokens") || name === "vocab.json" || name === "merges.txt";
|
|
492
492
|
const recognizedPayload = Boolean(name) && name !== "training_args.bin" && (name.endsWith(".safetensors") || name.endsWith(".bin"));
|
|
493
493
|
const adapterWeight = name === "adapter_model.safetensors" || name === "adapter_model.bin";
|
|
494
494
|
const fullModelWeight = Boolean(name) && (name.endsWith(".safetensors") && !name.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(name));
|
|
495
|
-
if (size > 0 && name === "adapter_config.json" && (
|
|
495
|
+
if (size > 0 && name === "adapter_config.json" && (type2 === "\0" || type2 === "0")) {
|
|
496
496
|
adapterConfigEntries += 1;
|
|
497
497
|
}
|
|
498
|
-
if ((
|
|
498
|
+
if ((type2 === "x" || type2 === "g") && size > maxPaxBytes) {
|
|
499
499
|
throw new Error(`PAX metadata exceeds 1 MiB in ${path}.`);
|
|
500
500
|
}
|
|
501
|
-
if (size > 0 && !metadataOnly && (
|
|
502
|
-
if (size > 0 && recognizedPayload && (
|
|
501
|
+
if (size > 0 && !metadataOnly && (type2 === "\0" || type2 === "0")) payloadEntries += 1;
|
|
502
|
+
if (size > 0 && recognizedPayload && (type2 === "\0" || type2 === "0")) {
|
|
503
503
|
recognizedPayloadEntries += 1;
|
|
504
504
|
recognizedPayloadBytes += size;
|
|
505
505
|
}
|
|
506
|
-
if (size > 0 && adapterWeight && (
|
|
506
|
+
if (size > 0 && adapterWeight && (type2 === "\0" || type2 === "0")) {
|
|
507
507
|
adapterWeightEntries += 1;
|
|
508
508
|
adapterWeightBytes += size;
|
|
509
509
|
}
|
|
510
|
-
if (size > 0 && fullModelWeight && (
|
|
510
|
+
if (size > 0 && fullModelWeight && (type2 === "\0" || type2 === "0")) {
|
|
511
511
|
fullModelWeightEntries += 1;
|
|
512
512
|
fullModelWeightBytes += size;
|
|
513
513
|
}
|
|
@@ -515,7 +515,7 @@ async function verifyTarGzipArchive(path) {
|
|
|
515
515
|
if (expandedBytes > maxExpandedBytes) throw new Error(`Tar archive exceeds 20 GiB expanded size: ${path}.`);
|
|
516
516
|
payloadBytes = size;
|
|
517
517
|
paddingBytes = (512 - size % 512) % 512;
|
|
518
|
-
paxType =
|
|
518
|
+
paxType = type2 === "x" || type2 === "g" ? type2 : null;
|
|
519
519
|
paxChunks = [];
|
|
520
520
|
entries += 1;
|
|
521
521
|
if (entries > maxEntries) throw new Error(`Tar archive exceeds ${maxEntries} members: ${path}.`);
|
|
@@ -834,28 +834,371 @@ async function assertUsableModelArtifact(uri) {
|
|
|
834
834
|
return inspection;
|
|
835
835
|
}
|
|
836
836
|
|
|
837
|
-
// src/local-runtime/
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
837
|
+
// src/local-runtime/run-reporter.ts
|
|
838
|
+
function reportInBackground(action) {
|
|
839
|
+
try {
|
|
840
|
+
void Promise.resolve(action()).catch(() => void 0);
|
|
841
|
+
} catch {
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
function sanitizeLogLine(line) {
|
|
845
|
+
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
846
|
+
}
|
|
847
|
+
function forwardStreamLines(stream, onLine) {
|
|
848
|
+
let buffered = "";
|
|
849
|
+
stream.setEncoding("utf8");
|
|
850
|
+
stream.on("data", (chunk) => {
|
|
851
|
+
buffered += chunk;
|
|
852
|
+
const lines = buffered.split(/\r\n|\n|\r/);
|
|
853
|
+
buffered = lines.pop() ?? "";
|
|
854
|
+
for (const line of lines) {
|
|
855
|
+
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
856
|
+
}
|
|
857
|
+
});
|
|
858
|
+
stream.on("end", () => {
|
|
859
|
+
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
860
|
+
});
|
|
861
|
+
}
|
|
841
862
|
|
|
842
|
-
// src/
|
|
863
|
+
// src/local-runtime/package-root.ts
|
|
843
864
|
import { existsSync } from "fs";
|
|
865
|
+
import { dirname as dirname2, resolve as resolve3 } from "path";
|
|
866
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
867
|
+
function localRuntimePackageRoot(moduleUrl) {
|
|
868
|
+
const moduleDirectory = dirname2(fileURLToPath2(moduleUrl));
|
|
869
|
+
const candidates = [
|
|
870
|
+
dirname2(moduleDirectory),
|
|
871
|
+
dirname2(dirname2(moduleDirectory))
|
|
872
|
+
];
|
|
873
|
+
const root = candidates.find(
|
|
874
|
+
(candidate) => existsSync(resolve3(candidate, "package.json")) && existsSync(resolve3(candidate, "training/adapter"))
|
|
875
|
+
);
|
|
876
|
+
if (!root) {
|
|
877
|
+
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
878
|
+
}
|
|
879
|
+
return root;
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
// src/local-runtime/capability-profiles.ts
|
|
883
|
+
var GIB = 1024 ** 3;
|
|
884
|
+
function gib(n) {
|
|
885
|
+
return Math.round(n * GIB);
|
|
886
|
+
}
|
|
887
|
+
var FOUNDATION_DEFAULT_DEPTH = 2;
|
|
888
|
+
var FOUNDATION_MAX_DEPTH = 64;
|
|
889
|
+
var FOUNDATION_DEFAULT_VOCAB = 256;
|
|
890
|
+
var FOUNDATION_DEFAULT_SEQUENCE = 64;
|
|
891
|
+
var FOUNDATION_DEFAULT_BATCH = 2;
|
|
892
|
+
var ADAPTER_MEMORY_PROFILES = [
|
|
893
|
+
{
|
|
894
|
+
id: "Qwen/Qwen3.5-2B",
|
|
895
|
+
diskBytes: gib(8),
|
|
896
|
+
inferenceBytes: gib(8),
|
|
897
|
+
trainBytes: gib(16),
|
|
898
|
+
sparkClass: false
|
|
899
|
+
},
|
|
900
|
+
{
|
|
901
|
+
id: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
|
|
902
|
+
diskBytes: gib(80),
|
|
903
|
+
inferenceBytes: gib(80),
|
|
904
|
+
trainBytes: gib(96),
|
|
905
|
+
sparkClass: true
|
|
906
|
+
},
|
|
907
|
+
{
|
|
908
|
+
id: "meta-models/Muse-Glimmer-30B",
|
|
909
|
+
diskBytes: gib(80),
|
|
910
|
+
inferenceBytes: gib(80),
|
|
911
|
+
trainBytes: gib(96),
|
|
912
|
+
sparkClass: true
|
|
913
|
+
}
|
|
914
|
+
];
|
|
915
|
+
function adapterMemoryProfile(modelId) {
|
|
916
|
+
const normalized = modelId.trim().toLowerCase();
|
|
917
|
+
return ADAPTER_MEMORY_PROFILES.find((profile) => profile.id.toLowerCase() === normalized);
|
|
918
|
+
}
|
|
919
|
+
function foundationWidth(depth) {
|
|
920
|
+
return 32 * depth;
|
|
921
|
+
}
|
|
922
|
+
function foundationParamCount(depth, vocabSize = FOUNDATION_DEFAULT_VOCAB, sequenceLength = FOUNDATION_DEFAULT_SEQUENCE) {
|
|
923
|
+
const width = foundationWidth(depth);
|
|
924
|
+
return 12 * depth * width * width + vocabSize * width + sequenceLength * width;
|
|
925
|
+
}
|
|
926
|
+
function foundationTrainBytes(depth, options = {}) {
|
|
927
|
+
const vocabSize = options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB;
|
|
928
|
+
const sequenceLength = options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE;
|
|
929
|
+
const batchSize = options.batchSize ?? FOUNDATION_DEFAULT_BATCH;
|
|
930
|
+
const params = foundationParamCount(depth, vocabSize, sequenceLength);
|
|
931
|
+
const width = foundationWidth(depth);
|
|
932
|
+
const activationSlack = batchSize * sequenceLength * width * depth * 64;
|
|
933
|
+
return 16 * params + activationSlack;
|
|
934
|
+
}
|
|
935
|
+
function foundationInferenceBytes(depth, options = {}) {
|
|
936
|
+
const params = foundationParamCount(
|
|
937
|
+
depth,
|
|
938
|
+
options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB,
|
|
939
|
+
options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE
|
|
940
|
+
);
|
|
941
|
+
return 4 * params + gib(0.25);
|
|
942
|
+
}
|
|
943
|
+
function isUnifiedGpuName(name) {
|
|
944
|
+
if (!name) return false;
|
|
945
|
+
return /spark|gb10/i.test(name);
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
// src/local-runtime/capability.ts
|
|
949
|
+
var HEADROOM = 1.2;
|
|
950
|
+
var FIT_RATIO = 0.7;
|
|
951
|
+
function formatGiB(bytes) {
|
|
952
|
+
const amount = bytes / 1024 ** 3;
|
|
953
|
+
const precision = amount >= 10 ? 0 : 1;
|
|
954
|
+
return `${amount.toFixed(precision)} GiB`;
|
|
955
|
+
}
|
|
956
|
+
function compareFit(available, required, missing) {
|
|
957
|
+
if (available === void 0) {
|
|
958
|
+
return { status: "not_possible", reason: missing, required_bytes: required };
|
|
959
|
+
}
|
|
960
|
+
if (available < required) {
|
|
961
|
+
return {
|
|
962
|
+
status: "not_possible",
|
|
963
|
+
reason: `needs ~${formatGiB(required)}; ${formatGiB(available)} available`,
|
|
964
|
+
required_bytes: required,
|
|
965
|
+
available_bytes: available
|
|
966
|
+
};
|
|
967
|
+
}
|
|
968
|
+
if (available < required * HEADROOM) {
|
|
969
|
+
return {
|
|
970
|
+
status: "tight",
|
|
971
|
+
reason: `fits in ${formatGiB(available)} with little headroom (needs ~${formatGiB(required)})`,
|
|
972
|
+
required_bytes: required,
|
|
973
|
+
available_bytes: available
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
return {
|
|
977
|
+
status: "ready",
|
|
978
|
+
reason: `${formatGiB(available)} available (needs ~${formatGiB(required)})`,
|
|
979
|
+
required_bytes: required,
|
|
980
|
+
available_bytes: available
|
|
981
|
+
};
|
|
982
|
+
}
|
|
983
|
+
function trainingBlockers(inventory) {
|
|
984
|
+
const blockers = [];
|
|
985
|
+
if (!inventory.node.ok) blockers.push(`Node ${inventory.node.version} is below 22`);
|
|
986
|
+
if (inventory.uv && !inventory.uv.ok) blockers.push(inventory.uv.message);
|
|
987
|
+
if (inventory.python && !inventory.python.ok) blockers.push(inventory.python.message);
|
|
988
|
+
return blockers;
|
|
989
|
+
}
|
|
990
|
+
function cudaAvailable(inventory) {
|
|
991
|
+
if (inventory.python?.ok && inventory.python.cuda_available !== void 0) {
|
|
992
|
+
return inventory.python.cuda_available;
|
|
993
|
+
}
|
|
994
|
+
return inventory.nvidia_smi.ok && inventory.gpus.length > 0;
|
|
995
|
+
}
|
|
996
|
+
function trainingGpu(inventory) {
|
|
997
|
+
const listed = inventory.gpus.find((gpu) => gpu.index === 0) ?? inventory.gpus[0];
|
|
998
|
+
if (listed) {
|
|
999
|
+
if (!listed.memory_total_bytes && inventory.python?.total_memory_bytes) {
|
|
1000
|
+
return { ...listed, memory_total_bytes: inventory.python.total_memory_bytes };
|
|
1001
|
+
}
|
|
1002
|
+
return listed;
|
|
1003
|
+
}
|
|
1004
|
+
if (inventory.python?.cuda_device) {
|
|
1005
|
+
return {
|
|
1006
|
+
index: 0,
|
|
1007
|
+
name: inventory.python.cuda_device,
|
|
1008
|
+
memory_total_bytes: inventory.python.total_memory_bytes,
|
|
1009
|
+
unified_memory: isUnifiedGpuName(inventory.python.cuda_device)
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
return void 0;
|
|
1013
|
+
}
|
|
1014
|
+
function gpuMemory(gpu) {
|
|
1015
|
+
return gpu?.memory_total_bytes;
|
|
1016
|
+
}
|
|
1017
|
+
function diskForCache(inventory) {
|
|
1018
|
+
return inventory.disks.find((disk) => disk.name === "model-cache")?.free_bytes;
|
|
1019
|
+
}
|
|
1020
|
+
function worse(left, right) {
|
|
1021
|
+
const rank = { not_possible: 2, tight: 1, ready: 0 };
|
|
1022
|
+
return rank[left.status] >= rank[right.status] ? left : right;
|
|
1023
|
+
}
|
|
1024
|
+
function withBlockers(verdict, blockers) {
|
|
1025
|
+
if (blockers.length === 0) return verdict;
|
|
1026
|
+
return {
|
|
1027
|
+
status: "not_possible",
|
|
1028
|
+
reason: blockers.join("; "),
|
|
1029
|
+
required_bytes: verdict.required_bytes,
|
|
1030
|
+
available_bytes: verdict.available_bytes
|
|
1031
|
+
};
|
|
1032
|
+
}
|
|
1033
|
+
function adapterVerdicts(profile, inventory, gpu, hasCuda) {
|
|
1034
|
+
const blockers = trainingBlockers(inventory);
|
|
1035
|
+
const gpuBytes = gpuMemory(gpu);
|
|
1036
|
+
const ram = inventory.os.total_memory_bytes;
|
|
1037
|
+
const disk = diskForCache(inventory);
|
|
1038
|
+
let train;
|
|
1039
|
+
if (!hasCuda) {
|
|
1040
|
+
train = {
|
|
1041
|
+
status: "not_possible",
|
|
1042
|
+
reason: "LoRA training requires CUDA",
|
|
1043
|
+
required_bytes: profile.trainBytes
|
|
1044
|
+
};
|
|
1045
|
+
} else {
|
|
1046
|
+
train = compareFit(gpuBytes, profile.trainBytes, "GPU memory is unknown");
|
|
1047
|
+
}
|
|
1048
|
+
train = withBlockers(train, blockers);
|
|
1049
|
+
if (disk !== void 0 && disk < profile.diskBytes && train.status !== "not_possible") {
|
|
1050
|
+
train = worse(train, compareFit(disk, profile.diskBytes, "model cache disk is unknown"));
|
|
1051
|
+
}
|
|
1052
|
+
const finetune = train;
|
|
1053
|
+
let inference;
|
|
1054
|
+
if (hasCuda) {
|
|
1055
|
+
inference = compareFit(gpuBytes, profile.inferenceBytes, "GPU memory is unknown");
|
|
1056
|
+
} else {
|
|
1057
|
+
inference = compareFit(ram, profile.inferenceBytes, "system RAM is unknown");
|
|
1058
|
+
if (inference.status !== "not_possible") {
|
|
1059
|
+
inference = {
|
|
1060
|
+
...inference,
|
|
1061
|
+
reason: `CPU inference: ${inference.reason}`
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
return {
|
|
1066
|
+
id: profile.id,
|
|
1067
|
+
spark_class: profile.sparkClass,
|
|
1068
|
+
train,
|
|
1069
|
+
finetune,
|
|
1070
|
+
inference
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
function suggestedFoundationDepth(availableBytes) {
|
|
1074
|
+
if (availableBytes === void 0) return FOUNDATION_DEFAULT_DEPTH;
|
|
1075
|
+
const budget = availableBytes * FIT_RATIO;
|
|
1076
|
+
let suggested = FOUNDATION_DEFAULT_DEPTH;
|
|
1077
|
+
for (let depth = FOUNDATION_DEFAULT_DEPTH; depth <= FOUNDATION_MAX_DEPTH; depth += 1) {
|
|
1078
|
+
if (foundationTrainBytes(depth) <= budget) suggested = depth;
|
|
1079
|
+
else break;
|
|
1080
|
+
}
|
|
1081
|
+
return suggested;
|
|
1082
|
+
}
|
|
1083
|
+
function evaluateCapabilities(inventory) {
|
|
1084
|
+
const hasCuda = cudaAvailable(inventory);
|
|
1085
|
+
const gpu = trainingGpu(inventory);
|
|
1086
|
+
const notes = [];
|
|
1087
|
+
if (gpu?.unified_memory || isUnifiedGpuName(gpu?.name)) {
|
|
1088
|
+
notes.push("GPU memory is a unified pool shared with system RAM (Spark-class).");
|
|
1089
|
+
}
|
|
1090
|
+
if (inventory.quick) {
|
|
1091
|
+
notes.push("Quick probe skipped the bundled torch/uv runtime.");
|
|
1092
|
+
}
|
|
1093
|
+
if (!inventory.node.ok) {
|
|
1094
|
+
notes.push(`Node ${inventory.node.version} is below the 22.x required by the local runner.`);
|
|
1095
|
+
}
|
|
1096
|
+
const adapters = ADAPTER_MEMORY_PROFILES.map(
|
|
1097
|
+
(profile) => adapterVerdicts(profile, inventory, gpu, hasCuda)
|
|
1098
|
+
);
|
|
1099
|
+
const blockers = trainingBlockers(inventory);
|
|
1100
|
+
const gpuBytes = gpuMemory(gpu);
|
|
1101
|
+
const ram = inventory.os.total_memory_bytes;
|
|
1102
|
+
const trainBudget = hasCuda ? gpuBytes : void 0;
|
|
1103
|
+
const suggested = suggestedFoundationDepth(trainBudget);
|
|
1104
|
+
const defaultTrain = foundationTrainBytes(FOUNDATION_DEFAULT_DEPTH);
|
|
1105
|
+
const inferBudget = hasCuda ? gpuBytes : ram;
|
|
1106
|
+
let train;
|
|
1107
|
+
if (!hasCuda) {
|
|
1108
|
+
train = {
|
|
1109
|
+
status: "not_possible",
|
|
1110
|
+
reason: "Foundation pretrain, SFT, and RL require CUDA",
|
|
1111
|
+
required_bytes: defaultTrain
|
|
1112
|
+
};
|
|
1113
|
+
} else {
|
|
1114
|
+
train = compareFit(gpuBytes, defaultTrain, "GPU memory is unknown");
|
|
1115
|
+
}
|
|
1116
|
+
train = withBlockers(train, blockers);
|
|
1117
|
+
const inference = compareFit(
|
|
1118
|
+
inferBudget,
|
|
1119
|
+
foundationInferenceBytes(FOUNDATION_DEFAULT_DEPTH),
|
|
1120
|
+
hasCuda ? "GPU memory is unknown" : "system RAM is unknown"
|
|
1121
|
+
);
|
|
1122
|
+
const foundation = {
|
|
1123
|
+
default_depth: FOUNDATION_DEFAULT_DEPTH,
|
|
1124
|
+
suggested_max_depth: hasCuda && blockers.length === 0 ? suggested : FOUNDATION_DEFAULT_DEPTH,
|
|
1125
|
+
train,
|
|
1126
|
+
finetune: train,
|
|
1127
|
+
inference: hasCuda ? inference : { ...inference, reason: `CPU eval: ${inference.reason}` },
|
|
1128
|
+
serve: {
|
|
1129
|
+
status: "not_possible",
|
|
1130
|
+
reason: "tt serve cannot host foundation checkpoints yet"
|
|
1131
|
+
}
|
|
1132
|
+
};
|
|
1133
|
+
return { cuda_available: hasCuda, gpu, adapters, foundation, notes };
|
|
1134
|
+
}
|
|
1135
|
+
function formatCapabilitySummary(report) {
|
|
1136
|
+
const gpu = report.gpu;
|
|
1137
|
+
const gpuLabel = gpu ? `${gpu.name}${gpu.memory_total_bytes ? `, ${formatGiB(gpu.memory_total_bytes)}` : ""}${gpu.unified_memory ? " unified" : ""}` : "no NVIDIA GPU";
|
|
1138
|
+
const adapterBits = report.adapters.map((adapter) => {
|
|
1139
|
+
const short = adapter.id.split("/").at(-1) ?? adapter.id;
|
|
1140
|
+
return `${short} train ${adapter.train.status}`;
|
|
1141
|
+
});
|
|
1142
|
+
return [
|
|
1143
|
+
`GPU ${gpuLabel}, CUDA ${report.cuda_available ? "yes" : "no"}`,
|
|
1144
|
+
`Adapter: ${adapterBits.join("; ")}`,
|
|
1145
|
+
`Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve not supported`
|
|
1146
|
+
].join(". ");
|
|
1147
|
+
}
|
|
1148
|
+
function warningsFromSnapshot(report, target) {
|
|
1149
|
+
const warnings = [];
|
|
1150
|
+
if (target.engine === "foundation") {
|
|
1151
|
+
for (const kind of ["train", "finetune"]) {
|
|
1152
|
+
const verdict = report.foundation[kind];
|
|
1153
|
+
if (verdict.status === "not_possible" || verdict.status === "tight") {
|
|
1154
|
+
warnings.push(`Foundation ${kind} is ${verdict.status}: ${verdict.reason}`);
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
return warnings;
|
|
1158
|
+
}
|
|
1159
|
+
const modelId = target.baseModel;
|
|
1160
|
+
const adapter = modelId ? report.adapters.find((item) => item.id.toLowerCase() === modelId.trim().toLowerCase()) : void 0;
|
|
1161
|
+
if (!adapter) {
|
|
1162
|
+
if (modelId && !adapterMemoryProfile(modelId)) return warnings;
|
|
1163
|
+
const blocked = report.adapters.filter((item) => item.train.status === "not_possible");
|
|
1164
|
+
if (blocked.length === report.adapters.length) {
|
|
1165
|
+
warnings.push(`Adapter train is not_possible: ${report.adapters[0]?.train.reason ?? "CUDA required"}`);
|
|
1166
|
+
}
|
|
1167
|
+
return warnings;
|
|
1168
|
+
}
|
|
1169
|
+
for (const kind of ["train", "finetune", "inference"]) {
|
|
1170
|
+
const verdict = adapter[kind];
|
|
1171
|
+
if (verdict.status === "not_possible" || verdict.status === "tight") {
|
|
1172
|
+
warnings.push(`${adapter.id} ${kind} is ${verdict.status}: ${verdict.reason}`);
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
return warnings;
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
// src/local-runtime/host-inventory.ts
|
|
1179
|
+
import { constants } from "fs";
|
|
1180
|
+
import { access, statfs } from "fs/promises";
|
|
1181
|
+
import { cpus, freemem, homedir as homedir3, totalmem, type } from "os";
|
|
1182
|
+
import { dirname as dirname5, join as join5, resolve as resolve7 } from "path";
|
|
1183
|
+
import { spawn as spawn2 } from "child_process";
|
|
1184
|
+
|
|
1185
|
+
// src/paths.ts
|
|
1186
|
+
import { existsSync as existsSync2 } from "fs";
|
|
844
1187
|
import { homedir } from "os";
|
|
845
|
-
import { isAbsolute as isAbsolute2, join as join2, resolve as
|
|
1188
|
+
import { isAbsolute as isAbsolute2, join as join2, resolve as resolve4 } from "path";
|
|
846
1189
|
var TUNED_TENSOR_DIR = ".tuned-tensor";
|
|
847
1190
|
var DEFAULT_ARTIFACT_ROOT = `${TUNED_TENSOR_DIR}/artifacts`;
|
|
848
1191
|
var DEFAULT_PROJECT_STORE_ROOT = `${TUNED_TENSOR_DIR}/store`;
|
|
849
1192
|
var DEFAULT_FOUNDATION_RUNS_DIR = `${TUNED_TENSOR_DIR}/foundation-runs`;
|
|
850
1193
|
function envHome(env = process.env) {
|
|
851
1194
|
const home = env.HOME?.trim();
|
|
852
|
-
return home ?
|
|
1195
|
+
return home ? resolve4(home) : homedir();
|
|
853
1196
|
}
|
|
854
1197
|
function expandUserPath(value, env = process.env, baseDirectory = process.cwd()) {
|
|
855
1198
|
const trimmed = value.trim();
|
|
856
1199
|
if (trimmed === "~") return envHome(env);
|
|
857
|
-
if (trimmed.startsWith("~/")) return
|
|
858
|
-
return isAbsolute2(trimmed) ?
|
|
1200
|
+
if (trimmed.startsWith("~/")) return resolve4(envHome(env), trimmed.slice(2));
|
|
1201
|
+
return isAbsolute2(trimmed) ? resolve4(trimmed) : resolve4(baseDirectory, trimmed);
|
|
859
1202
|
}
|
|
860
1203
|
function getTunedTensorHome(env = process.env) {
|
|
861
1204
|
const override = env.TUNED_TENSOR_HOME?.trim();
|
|
@@ -863,14 +1206,14 @@ function getTunedTensorHome(env = process.env) {
|
|
|
863
1206
|
return join2(envHome(env), TUNED_TENSOR_DIR);
|
|
864
1207
|
}
|
|
865
1208
|
function legacyConfigDir(env) {
|
|
866
|
-
const configHome = env.XDG_CONFIG_HOME?.trim() ?
|
|
1209
|
+
const configHome = env.XDG_CONFIG_HOME?.trim() ? resolve4(env.XDG_CONFIG_HOME) : join2(envHome(env), ".config");
|
|
867
1210
|
return join2(configHome, "tuned-tensor");
|
|
868
1211
|
}
|
|
869
1212
|
function getConfigDir(env = process.env) {
|
|
870
1213
|
const next = getTunedTensorHome(env);
|
|
871
1214
|
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
872
1215
|
const legacy = legacyConfigDir(env);
|
|
873
|
-
if (!
|
|
1216
|
+
if (!existsSync2(join2(next, "config.json")) && existsSync2(join2(legacy, "config.json"))) {
|
|
874
1217
|
return legacy;
|
|
875
1218
|
}
|
|
876
1219
|
return next;
|
|
@@ -879,7 +1222,7 @@ function getAgentConfigDir(env = process.env) {
|
|
|
879
1222
|
const next = join2(getTunedTensorHome(env), "agent");
|
|
880
1223
|
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
881
1224
|
const legacy = join2(legacyConfigDir(env), "agent");
|
|
882
|
-
if (!
|
|
1225
|
+
if (!existsSync2(next) && existsSync2(legacy)) return legacy;
|
|
883
1226
|
return next;
|
|
884
1227
|
}
|
|
885
1228
|
function defaultStoreRoot(env = process.env) {
|
|
@@ -888,7 +1231,7 @@ function defaultStoreRoot(env = process.env) {
|
|
|
888
1231
|
const next = join2(getTunedTensorHome(env), "store");
|
|
889
1232
|
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
890
1233
|
const legacy = join2(envHome(env), ".tuned-tensor-local");
|
|
891
|
-
if (!
|
|
1234
|
+
if (!existsSync2(next) && existsSync2(legacy)) return legacy;
|
|
892
1235
|
return next;
|
|
893
1236
|
}
|
|
894
1237
|
function defaultCacheRoot(env = process.env) {
|
|
@@ -897,164 +1240,871 @@ function defaultCacheRoot(env = process.env) {
|
|
|
897
1240
|
function pythonEnvironmentPath(kind, hash, env = process.env) {
|
|
898
1241
|
const next = join2(defaultCacheRoot(env), kind, hash);
|
|
899
1242
|
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
900
|
-
const cacheHome = env.XDG_CACHE_HOME?.trim() ?
|
|
1243
|
+
const cacheHome = env.XDG_CACHE_HOME?.trim() ? resolve4(env.XDG_CACHE_HOME) : join2(envHome(env), ".cache");
|
|
901
1244
|
const legacy = join2(cacheHome, "tuned-tensor-local", kind, hash);
|
|
902
|
-
if (!
|
|
1245
|
+
if (!existsSync2(next) && existsSync2(legacy)) return legacy;
|
|
903
1246
|
return next;
|
|
904
1247
|
}
|
|
905
1248
|
|
|
906
|
-
// src/local-runtime/
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
1249
|
+
// src/local-runtime/huggingface-cache.ts
|
|
1250
|
+
import { homedir as homedir2 } from "os";
|
|
1251
|
+
import { dirname as dirname3, join as join3, resolve as resolve5 } from "path";
|
|
1252
|
+
function minimalMachineLearningEnvironment(env) {
|
|
1253
|
+
const exact = /* @__PURE__ */ new Set([
|
|
1254
|
+
"PATH",
|
|
1255
|
+
"HOME",
|
|
1256
|
+
"USER",
|
|
1257
|
+
"LOGNAME",
|
|
1258
|
+
"SHELL",
|
|
1259
|
+
"TMPDIR",
|
|
1260
|
+
"TEMP",
|
|
1261
|
+
"TMP",
|
|
1262
|
+
"LANG",
|
|
1263
|
+
"LANGUAGE",
|
|
1264
|
+
"LD_LIBRARY_PATH",
|
|
1265
|
+
"DYLD_LIBRARY_PATH",
|
|
1266
|
+
"XDG_CACHE_HOME",
|
|
1267
|
+
"SSL_CERT_FILE",
|
|
1268
|
+
"REQUESTS_CA_BUNDLE",
|
|
1269
|
+
"CURL_CA_BUNDLE",
|
|
1270
|
+
"NO_PROXY",
|
|
1271
|
+
"no_proxy",
|
|
1272
|
+
"UV_CACHE_DIR",
|
|
1273
|
+
"UV_PYTHON",
|
|
1274
|
+
"HF_HOME",
|
|
1275
|
+
"HF_HUB_CACHE",
|
|
1276
|
+
"HUGGINGFACE_HUB_CACHE",
|
|
1277
|
+
"HF_ENDPOINT",
|
|
1278
|
+
"HF_TOKEN",
|
|
1279
|
+
"HF_HUB_OFFLINE",
|
|
1280
|
+
"HF_HUB_DISABLE_TELEMETRY",
|
|
1281
|
+
"TRANSFORMERS_OFFLINE",
|
|
1282
|
+
"HF_HUB_ENABLE_HF_TRANSFER",
|
|
1283
|
+
"PYTHONUTF8",
|
|
1284
|
+
"PYTHONIOENCODING"
|
|
1285
|
+
]);
|
|
1286
|
+
const prefixes = [
|
|
1287
|
+
"LC_",
|
|
1288
|
+
"CUDA_",
|
|
1289
|
+
"NVIDIA_",
|
|
1290
|
+
"NCCL_",
|
|
1291
|
+
"OMP_",
|
|
1292
|
+
"MKL_",
|
|
1293
|
+
"TORCH_",
|
|
1294
|
+
"PYTORCH_",
|
|
1295
|
+
"TOKENIZERS_",
|
|
1296
|
+
"HF_XET_"
|
|
1297
|
+
];
|
|
1298
|
+
const result = Object.fromEntries(Object.entries(env).filter(
|
|
1299
|
+
([key]) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix))
|
|
1300
|
+
));
|
|
1301
|
+
result.TORCH_DISABLE_NATIVE_JIT ??= "1";
|
|
1302
|
+
return result;
|
|
912
1303
|
}
|
|
913
|
-
function
|
|
1304
|
+
function resolveHuggingFaceCacheLayout(modelCache) {
|
|
1305
|
+
const hfHome = resolve5(modelCache);
|
|
914
1306
|
return {
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
runsDir: join3(root, "runs"),
|
|
918
|
-
modelsDir: join3(root, "models"),
|
|
919
|
-
datasetsDir: join3(root, "datasets")
|
|
1307
|
+
hfHome,
|
|
1308
|
+
hubCache: join3(hfHome, "hub")
|
|
920
1309
|
};
|
|
921
1310
|
}
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
if (!metadata.isDirectory()) {
|
|
937
|
-
throw new Error(`Expected local store directory: ${path}`);
|
|
938
|
-
}
|
|
939
|
-
await chmod(path, 448);
|
|
1311
|
+
function withHuggingFaceCacheEnvironment(env, modelCache) {
|
|
1312
|
+
const inheritedHub = env.HF_HUB_CACHE ?? env.HUGGINGFACE_HUB_CACHE;
|
|
1313
|
+
const canonicalHome = modelCache ?? env.HF_HOME ?? (inheritedHub ? dirname3(resolve5(inheritedHub)) : join3(homedir2(), ".cache", "huggingface"));
|
|
1314
|
+
const layout = resolveHuggingFaceCacheLayout(canonicalHome);
|
|
1315
|
+
const result = {
|
|
1316
|
+
...env,
|
|
1317
|
+
HF_HOME: layout.hfHome,
|
|
1318
|
+
HF_HUB_CACHE: layout.hubCache,
|
|
1319
|
+
HUGGINGFACE_HUB_CACHE: layout.hubCache
|
|
1320
|
+
};
|
|
1321
|
+
delete result.TRANSFORMERS_CACHE;
|
|
1322
|
+
delete result.PYTORCH_TRANSFORMERS_CACHE;
|
|
1323
|
+
delete result.PYTORCH_PRETRAINED_BERT_CACHE;
|
|
1324
|
+
return result;
|
|
940
1325
|
}
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
await chmod(path, 448);
|
|
948
|
-
for (const entry of await readdir3(path)) {
|
|
949
|
-
await hardenPrivateTree(join3(path, entry));
|
|
950
|
-
}
|
|
951
|
-
return;
|
|
952
|
-
}
|
|
953
|
-
if (metadata.isFile()) {
|
|
954
|
-
await chmod(path, 384);
|
|
955
|
-
return;
|
|
956
|
-
}
|
|
957
|
-
throw new Error(`Refusing unsupported filesystem entry in local store: ${path}`);
|
|
1326
|
+
function withOfflineHuggingFaceCacheEnvironment(env, modelCache) {
|
|
1327
|
+
return {
|
|
1328
|
+
...withHuggingFaceCacheEnvironment(env, modelCache),
|
|
1329
|
+
HF_HUB_OFFLINE: "1",
|
|
1330
|
+
TRANSFORMERS_OFFLINE: "1"
|
|
1331
|
+
};
|
|
958
1332
|
}
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
1333
|
+
|
|
1334
|
+
// src/local-runtime/process-runner.ts
|
|
1335
|
+
import { createHash as createHash2 } from "crypto";
|
|
1336
|
+
import { spawn } from "child_process";
|
|
1337
|
+
import { createWriteStream, readFileSync } from "fs";
|
|
1338
|
+
import { mkdir as mkdir2 } from "fs/promises";
|
|
1339
|
+
import { dirname as dirname4, join as join4, resolve as resolve6 } from "path";
|
|
1340
|
+
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
1341
|
+
var bundledProject = join4(packageRoot, "training/adapter");
|
|
1342
|
+
var bundledRuntimeHash = (() => {
|
|
1343
|
+
const hash = createHash2("sha256");
|
|
1344
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1345
|
+
hash.update(readFileSync(join4(bundledProject, name)));
|
|
970
1346
|
}
|
|
971
|
-
|
|
972
|
-
|
|
1347
|
+
return hash.digest("hex").slice(0, 20);
|
|
1348
|
+
})();
|
|
1349
|
+
var BUNDLED_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv", bundledRuntimeHash);
|
|
1350
|
+
var foundationProject = join4(packageRoot, "training/foundation");
|
|
1351
|
+
var foundationRuntimeHash = (() => {
|
|
1352
|
+
const hash = createHash2("sha256");
|
|
1353
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1354
|
+
hash.update(readFileSync(join4(foundationProject, name)));
|
|
973
1355
|
}
|
|
974
|
-
|
|
975
|
-
|
|
1356
|
+
return hash.digest("hex").slice(0, 20);
|
|
1357
|
+
})();
|
|
1358
|
+
var FOUNDATION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-foundation", foundationRuntimeHash);
|
|
1359
|
+
function buildBundledPythonCommand(entrypoint, args = []) {
|
|
1360
|
+
const target = entrypoint === "-c" ? entrypoint : join4(bundledProject, "src", entrypoint);
|
|
1361
|
+
const commandArgs = [
|
|
1362
|
+
"run",
|
|
1363
|
+
"--frozen",
|
|
1364
|
+
"--quiet",
|
|
1365
|
+
"--project",
|
|
1366
|
+
bundledProject,
|
|
1367
|
+
"python",
|
|
1368
|
+
target,
|
|
1369
|
+
...args
|
|
1370
|
+
];
|
|
1371
|
+
return {
|
|
1372
|
+
command: "uv",
|
|
1373
|
+
commandArgs,
|
|
1374
|
+
displayCommand: ["uv", ...commandArgs]
|
|
1375
|
+
};
|
|
976
1376
|
}
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
encoding: "utf8",
|
|
983
|
-
mode: 384
|
|
984
|
-
});
|
|
985
|
-
await rename2(tmp, path);
|
|
986
|
-
await chmod(path, 384);
|
|
1377
|
+
function withBundledPythonEnvironment(env) {
|
|
1378
|
+
return {
|
|
1379
|
+
...env,
|
|
1380
|
+
UV_PROJECT_ENVIRONMENT: BUNDLED_PYTHON_ENVIRONMENT
|
|
1381
|
+
};
|
|
987
1382
|
}
|
|
988
|
-
|
|
989
|
-
|
|
1383
|
+
function buildFoundationPythonCommand(entrypoint, args = []) {
|
|
1384
|
+
const commandArgs = [
|
|
1385
|
+
"run",
|
|
1386
|
+
"--frozen",
|
|
1387
|
+
"--quiet",
|
|
1388
|
+
"--project",
|
|
1389
|
+
foundationProject,
|
|
1390
|
+
"python",
|
|
1391
|
+
join4(foundationProject, "src", entrypoint),
|
|
1392
|
+
...args
|
|
1393
|
+
];
|
|
1394
|
+
return {
|
|
1395
|
+
command: "uv",
|
|
1396
|
+
commandArgs,
|
|
1397
|
+
displayCommand: ["uv", ...commandArgs]
|
|
1398
|
+
};
|
|
990
1399
|
}
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
1400
|
+
function withFoundationPythonEnvironment(env) {
|
|
1401
|
+
return {
|
|
1402
|
+
...env,
|
|
1403
|
+
UV_PROJECT_ENVIRONMENT: FOUNDATION_PYTHON_ENVIRONMENT
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1406
|
+
var ProcessCancelledError = class extends Error {
|
|
1407
|
+
constructor(message = "Process cancelled.") {
|
|
1408
|
+
super(message);
|
|
1409
|
+
this.name = "ProcessCancelledError";
|
|
1410
|
+
}
|
|
1411
|
+
};
|
|
1412
|
+
async function openProcessLog(path, exclusive) {
|
|
1413
|
+
const stream = createWriteStream(path, {
|
|
1414
|
+
flags: exclusive ? "wx" : "w",
|
|
996
1415
|
mode: 384
|
|
997
1416
|
});
|
|
998
|
-
await
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
await ensurePrivateDirectory(dirname2(to));
|
|
1012
|
-
await copyFile2(from, to);
|
|
1013
|
-
await chmod(to, 384);
|
|
1417
|
+
await new Promise((resolveOpen, reject) => {
|
|
1418
|
+
const onOpen = () => {
|
|
1419
|
+
stream.off("error", onError);
|
|
1420
|
+
resolveOpen();
|
|
1421
|
+
};
|
|
1422
|
+
const onError = (error) => {
|
|
1423
|
+
stream.off("open", onOpen);
|
|
1424
|
+
reject(error);
|
|
1425
|
+
};
|
|
1426
|
+
stream.once("open", onOpen);
|
|
1427
|
+
stream.once("error", onError);
|
|
1428
|
+
});
|
|
1429
|
+
return stream;
|
|
1014
1430
|
}
|
|
1015
|
-
async function
|
|
1431
|
+
async function runLoggedProcess(args) {
|
|
1432
|
+
if (args.logPath) await mkdir2(dirname4(args.logPath), { recursive: true });
|
|
1433
|
+
const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false) : null;
|
|
1434
|
+
let stderr = "";
|
|
1016
1435
|
try {
|
|
1017
|
-
const
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1436
|
+
const exitCode = await new Promise((resolvePromise, reject) => {
|
|
1437
|
+
const child = spawn(args.command, args.commandArgs, {
|
|
1438
|
+
cwd: args.cwd ? resolve6(args.cwd) : process.cwd(),
|
|
1439
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1440
|
+
env: args.env ?? process.env,
|
|
1441
|
+
detached: process.platform !== "win32"
|
|
1442
|
+
});
|
|
1443
|
+
let timedOut = false;
|
|
1444
|
+
let cancelled = false;
|
|
1445
|
+
let cancellationError;
|
|
1446
|
+
let cancellationCheckRunning = false;
|
|
1447
|
+
let forceKillTimer = null;
|
|
1448
|
+
const killProcessGroup = (signal) => {
|
|
1449
|
+
if (child.pid && process.platform !== "win32") {
|
|
1450
|
+
try {
|
|
1451
|
+
process.kill(-child.pid, signal);
|
|
1452
|
+
return;
|
|
1453
|
+
} catch {
|
|
1454
|
+
}
|
|
1455
|
+
}
|
|
1456
|
+
child.kill(signal);
|
|
1457
|
+
};
|
|
1458
|
+
const requestStop = (signal = "SIGTERM") => {
|
|
1459
|
+
killProcessGroup(signal);
|
|
1460
|
+
if (!forceKillTimer) {
|
|
1461
|
+
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), 5e3);
|
|
1462
|
+
forceKillTimer.unref();
|
|
1463
|
+
}
|
|
1464
|
+
};
|
|
1465
|
+
const onSigint = () => {
|
|
1466
|
+
cancelled = true;
|
|
1467
|
+
requestStop("SIGINT");
|
|
1468
|
+
};
|
|
1469
|
+
const onSigterm = () => {
|
|
1470
|
+
cancelled = true;
|
|
1471
|
+
requestStop("SIGTERM");
|
|
1472
|
+
};
|
|
1473
|
+
process.once("SIGINT", onSigint);
|
|
1474
|
+
process.once("SIGTERM", onSigterm);
|
|
1475
|
+
const timer = args.timeoutMs ? setTimeout(() => {
|
|
1476
|
+
timedOut = true;
|
|
1477
|
+
requestStop();
|
|
1478
|
+
}, args.timeoutMs) : null;
|
|
1479
|
+
const cancellationTimer = args.shouldCancel ? setInterval(() => {
|
|
1480
|
+
if (cancellationCheckRunning || cancelled) return;
|
|
1481
|
+
cancellationCheckRunning = true;
|
|
1482
|
+
Promise.resolve(args.shouldCancel?.()).then((requested) => {
|
|
1483
|
+
if (!requested || cancelled) return;
|
|
1484
|
+
cancelled = true;
|
|
1485
|
+
requestStop();
|
|
1486
|
+
}).catch((error) => {
|
|
1487
|
+
cancellationError = error;
|
|
1488
|
+
requestStop();
|
|
1489
|
+
}).finally(() => {
|
|
1490
|
+
cancellationCheckRunning = false;
|
|
1491
|
+
});
|
|
1492
|
+
}, args.cancelPollMs ?? 250) : null;
|
|
1493
|
+
cancellationTimer?.unref();
|
|
1494
|
+
const clearProcessTimers = () => {
|
|
1495
|
+
if (timer) clearTimeout(timer);
|
|
1496
|
+
if (cancellationTimer) clearInterval(cancellationTimer);
|
|
1497
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
1498
|
+
process.off("SIGINT", onSigint);
|
|
1499
|
+
process.off("SIGTERM", onSigterm);
|
|
1500
|
+
};
|
|
1501
|
+
if (logStream) {
|
|
1502
|
+
child.stdout.pipe(logStream, { end: false });
|
|
1503
|
+
child.stderr.pipe(logStream, { end: false });
|
|
1504
|
+
}
|
|
1505
|
+
forwardStreamLines(child.stdout, (line) => {
|
|
1506
|
+
args.onLine?.(line, "stdout");
|
|
1507
|
+
if (args.reporter?.verbose) {
|
|
1508
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stdout", message: line }));
|
|
1509
|
+
}
|
|
1510
|
+
});
|
|
1511
|
+
forwardStreamLines(child.stderr, (line) => {
|
|
1512
|
+
stderr += `${line}
|
|
1513
|
+
`;
|
|
1514
|
+
args.onLine?.(line, "stderr");
|
|
1515
|
+
if (args.reporter?.verbose) {
|
|
1516
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stderr", message: line }));
|
|
1517
|
+
}
|
|
1518
|
+
});
|
|
1519
|
+
child.on("error", (error) => {
|
|
1520
|
+
clearProcessTimers();
|
|
1521
|
+
reject(timedOut ? new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`) : error);
|
|
1522
|
+
});
|
|
1523
|
+
child.on("close", (code) => {
|
|
1524
|
+
if (timedOut || cancelled || cancellationError || args.terminateProcessGroupOnExit) {
|
|
1525
|
+
killProcessGroup("SIGKILL");
|
|
1526
|
+
}
|
|
1527
|
+
clearProcessTimers();
|
|
1528
|
+
if (timedOut) {
|
|
1529
|
+
reject(new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`));
|
|
1530
|
+
return;
|
|
1531
|
+
}
|
|
1532
|
+
if (cancellationError) {
|
|
1533
|
+
reject(cancellationError);
|
|
1534
|
+
return;
|
|
1535
|
+
}
|
|
1536
|
+
if (cancelled) {
|
|
1537
|
+
reject(new ProcessCancelledError(`${args.stage} was cancelled.`));
|
|
1538
|
+
return;
|
|
1539
|
+
}
|
|
1540
|
+
resolvePromise(code ?? 1);
|
|
1541
|
+
});
|
|
1542
|
+
});
|
|
1543
|
+
return { exitCode, stderr };
|
|
1544
|
+
} finally {
|
|
1545
|
+
if (logStream) {
|
|
1546
|
+
await new Promise((resolveEnd) => {
|
|
1547
|
+
logStream.end(resolveEnd);
|
|
1548
|
+
});
|
|
1549
|
+
}
|
|
1022
1550
|
}
|
|
1023
1551
|
}
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1552
|
+
|
|
1553
|
+
// src/local-runtime/host-inventory.ts
|
|
1554
|
+
async function runCommand(command, args, options = {}) {
|
|
1555
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
1556
|
+
let interrupted;
|
|
1557
|
+
const result = await new Promise((resolveResult) => {
|
|
1558
|
+
let settled = false;
|
|
1559
|
+
let stdout = "";
|
|
1560
|
+
let stderr = "";
|
|
1561
|
+
let stopError;
|
|
1562
|
+
let forceKillTimer = null;
|
|
1563
|
+
const child = spawn2(command, args, {
|
|
1564
|
+
cwd: options.cwd,
|
|
1565
|
+
env: options.env ?? process.env,
|
|
1566
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1567
|
+
detached: process.platform !== "win32"
|
|
1568
|
+
});
|
|
1569
|
+
const killProcessGroup = (signal) => {
|
|
1570
|
+
if (child.pid && process.platform !== "win32") {
|
|
1571
|
+
try {
|
|
1572
|
+
process.kill(-child.pid, signal);
|
|
1573
|
+
return;
|
|
1574
|
+
} catch {
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
child.kill(signal);
|
|
1578
|
+
};
|
|
1579
|
+
const requestStop = (signal = "SIGTERM") => {
|
|
1580
|
+
killProcessGroup(signal);
|
|
1581
|
+
if (!forceKillTimer) {
|
|
1582
|
+
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), 5e3);
|
|
1583
|
+
forceKillTimer.unref();
|
|
1584
|
+
}
|
|
1585
|
+
};
|
|
1586
|
+
const onSigint = () => {
|
|
1587
|
+
interrupted = "interrupted by SIGINT";
|
|
1588
|
+
stopError = interrupted;
|
|
1589
|
+
requestStop("SIGINT");
|
|
1590
|
+
};
|
|
1591
|
+
const onSigterm = () => {
|
|
1592
|
+
interrupted = "interrupted by SIGTERM";
|
|
1593
|
+
stopError = interrupted;
|
|
1594
|
+
requestStop("SIGTERM");
|
|
1595
|
+
};
|
|
1596
|
+
const finish = (result2) => {
|
|
1597
|
+
if (settled) return;
|
|
1598
|
+
settled = true;
|
|
1599
|
+
clearTimeout(timer);
|
|
1600
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
1601
|
+
process.off("SIGINT", onSigint);
|
|
1602
|
+
process.off("SIGTERM", onSigterm);
|
|
1603
|
+
resolveResult(result2);
|
|
1604
|
+
};
|
|
1605
|
+
const timer = setTimeout(() => {
|
|
1606
|
+
stopError = `timed out after ${timeoutMs}ms`;
|
|
1607
|
+
requestStop();
|
|
1608
|
+
}, timeoutMs);
|
|
1609
|
+
timer.unref();
|
|
1610
|
+
process.once("SIGINT", onSigint);
|
|
1611
|
+
process.once("SIGTERM", onSigterm);
|
|
1612
|
+
child.stdout.setEncoding("utf8");
|
|
1613
|
+
child.stderr.setEncoding("utf8");
|
|
1614
|
+
child.stdout.on("data", (chunk) => {
|
|
1615
|
+
stdout += chunk;
|
|
1616
|
+
});
|
|
1617
|
+
child.stderr.on("data", (chunk) => {
|
|
1618
|
+
stderr += chunk;
|
|
1619
|
+
});
|
|
1620
|
+
child.on("error", (error) => {
|
|
1621
|
+
finish({ code: null, stdout, stderr, error: error.message });
|
|
1622
|
+
});
|
|
1623
|
+
child.on("close", (code) => {
|
|
1624
|
+
finish({ code: stopError ? null : code, stdout, stderr, error: stopError });
|
|
1625
|
+
});
|
|
1626
|
+
});
|
|
1627
|
+
if (interrupted) throw new Error(interrupted);
|
|
1628
|
+
return result;
|
|
1027
1629
|
}
|
|
1028
|
-
function
|
|
1029
|
-
|
|
1030
|
-
return compareIds(left, right);
|
|
1630
|
+
function firstLine(value) {
|
|
1631
|
+
return value.trim().split(/\r?\n/)[0] ?? "";
|
|
1031
1632
|
}
|
|
1032
|
-
function
|
|
1033
|
-
if (
|
|
1034
|
-
|
|
1633
|
+
function parseMiB(value) {
|
|
1634
|
+
if (!value) return void 0;
|
|
1635
|
+
const parsed = Number(value.replace(/[^\d.]/g, ""));
|
|
1636
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
1637
|
+
return Math.round(parsed * 1024 * 1024);
|
|
1035
1638
|
}
|
|
1036
|
-
function
|
|
1037
|
-
return
|
|
1639
|
+
function isNvidiaSmiDecorativeLine(line) {
|
|
1640
|
+
return /^(?:mon|tue|wed|thu|fri|sat|sun)\b/i.test(line) || line.startsWith("+---") || line.startsWith("|");
|
|
1038
1641
|
}
|
|
1039
|
-
function
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1642
|
+
function gpuFromName(name, index, extras = {}) {
|
|
1643
|
+
return {
|
|
1644
|
+
index,
|
|
1645
|
+
name,
|
|
1646
|
+
unified_memory: /spark|gb10/i.test(name),
|
|
1647
|
+
...extras
|
|
1648
|
+
};
|
|
1043
1649
|
}
|
|
1044
|
-
function
|
|
1045
|
-
const
|
|
1046
|
-
const
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1650
|
+
function parseNvidiaGpus(stdout) {
|
|
1651
|
+
const gpus = [];
|
|
1652
|
+
for (const line of stdout.split(/\r?\n/).map((item) => item.trim()).filter(Boolean)) {
|
|
1653
|
+
if (isNvidiaSmiDecorativeLine(line)) continue;
|
|
1654
|
+
const parts = line.split(",").map((part) => part.trim());
|
|
1655
|
+
if (parts[0] && /^\d+$/.test(parts[0]) && parts.length >= 2) {
|
|
1656
|
+
const name = parts[1] || "NVIDIA GPU";
|
|
1657
|
+
gpus.push(gpuFromName(name, Number(parts[0]), {
|
|
1658
|
+
driver_version: parts[2] || void 0,
|
|
1659
|
+
memory_total_bytes: parseMiB(parts[3]),
|
|
1660
|
+
memory_free_bytes: parseMiB(parts[4]),
|
|
1661
|
+
compute_cap: parts[5] || void 0
|
|
1662
|
+
}));
|
|
1663
|
+
continue;
|
|
1664
|
+
}
|
|
1665
|
+
if (parts.length >= 2 && parts[0] && !/^(?:NVIDIA-SMI|Fail|Error)/i.test(parts[0])) {
|
|
1666
|
+
const name = parts[0];
|
|
1667
|
+
gpus.push(gpuFromName(name, gpus.length, {
|
|
1668
|
+
driver_version: parts[1] || void 0,
|
|
1669
|
+
memory_total_bytes: parseMiB(parts[2]),
|
|
1670
|
+
memory_free_bytes: parseMiB(parts[3])
|
|
1671
|
+
}));
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
return gpus;
|
|
1675
|
+
}
|
|
1676
|
+
function parseNvidiaTableGpus(stdout) {
|
|
1677
|
+
const gpus = [];
|
|
1678
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
1679
|
+
const match = line.match(/^\|\s+(\d+)\s+(.+?)\s+(?:On|Off|N\/A)\s+\|/);
|
|
1680
|
+
if (!match) continue;
|
|
1681
|
+
const name = (match[2] ?? "").trim();
|
|
1682
|
+
if (!name || /^NVIDIA-SMI/i.test(name)) continue;
|
|
1683
|
+
gpus.push(gpuFromName(name, Number(match[1])));
|
|
1684
|
+
}
|
|
1685
|
+
return gpus;
|
|
1686
|
+
}
|
|
1687
|
+
var NVIDIA_CSV_QUERIES = [
|
|
1688
|
+
["--query-gpu=index,name,driver_version,memory.total,memory.free", "--format=csv,noheader,nounits"],
|
|
1689
|
+
["--query-gpu=name,driver_version,memory.total,memory.free", "--format=csv,noheader,nounits"],
|
|
1690
|
+
["--query-gpu=name,driver_version", "--format=csv,noheader"]
|
|
1691
|
+
];
|
|
1692
|
+
async function queryNvidiaGpus(env) {
|
|
1693
|
+
let last;
|
|
1694
|
+
for (const args of NVIDIA_CSV_QUERIES) {
|
|
1695
|
+
const result = await runCommand("nvidia-smi", args, { env, timeoutMs: 1e4 });
|
|
1696
|
+
last = result;
|
|
1697
|
+
if (result.code !== 0) continue;
|
|
1698
|
+
const gpus2 = parseNvidiaGpus(result.stdout);
|
|
1699
|
+
if (gpus2.length > 0) return { result, gpus: gpus2 };
|
|
1700
|
+
}
|
|
1701
|
+
const table = await runCommand("nvidia-smi", [], { env, timeoutMs: 1e4 });
|
|
1702
|
+
const gpus = table.code === 0 ? parseNvidiaTableGpus(table.stdout) : [];
|
|
1703
|
+
return { result: table.code === 0 || !last ? table : last, gpus };
|
|
1704
|
+
}
|
|
1705
|
+
async function diskCheck(name, path) {
|
|
1706
|
+
const resolvedPath = resolve7(path);
|
|
1707
|
+
try {
|
|
1708
|
+
await access(resolvedPath, constants.R_OK);
|
|
1709
|
+
const fs = await statfs(resolvedPath);
|
|
1710
|
+
const freeBytes = Number(fs.bavail) * Number(fs.bsize);
|
|
1711
|
+
return {
|
|
1712
|
+
name,
|
|
1713
|
+
path: resolvedPath,
|
|
1714
|
+
free_bytes: freeBytes,
|
|
1715
|
+
ok: true,
|
|
1716
|
+
message: `${resolvedPath} (${Math.round(freeBytes / 1024 ** 3 * 10) / 10} GiB available)`
|
|
1717
|
+
};
|
|
1718
|
+
} catch (error) {
|
|
1719
|
+
let probe = resolvedPath;
|
|
1720
|
+
for (let i = 0; i < 6; i += 1) {
|
|
1721
|
+
const parent = dirname5(probe);
|
|
1722
|
+
if (parent === probe) break;
|
|
1723
|
+
probe = parent;
|
|
1724
|
+
try {
|
|
1725
|
+
const fs = await statfs(probe);
|
|
1726
|
+
const freeBytes = Number(fs.bavail) * Number(fs.bsize);
|
|
1727
|
+
return {
|
|
1728
|
+
name,
|
|
1729
|
+
path: resolvedPath,
|
|
1730
|
+
free_bytes: freeBytes,
|
|
1731
|
+
ok: true,
|
|
1732
|
+
message: `${resolvedPath} not created yet; ${probe} has ${Math.round(freeBytes / 1024 ** 3 * 10) / 10} GiB available`
|
|
1733
|
+
};
|
|
1734
|
+
} catch {
|
|
1735
|
+
continue;
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
return {
|
|
1739
|
+
name,
|
|
1740
|
+
path: resolvedPath,
|
|
1741
|
+
ok: false,
|
|
1742
|
+
message: `${resolvedPath}: ${error instanceof Error ? error.message : String(error)}`
|
|
1743
|
+
};
|
|
1744
|
+
}
|
|
1745
|
+
}
|
|
1746
|
+
function pythonProbeSource() {
|
|
1747
|
+
return [
|
|
1748
|
+
"import json",
|
|
1749
|
+
"import torch, transformers, peft, huggingface_hub",
|
|
1750
|
+
"payload = {'python_ok': True, 'torch': torch.__version__, 'transformers': transformers.__version__, 'peft': getattr(peft, '__version__', None), 'cuda_available': bool(torch.cuda.is_available())}",
|
|
1751
|
+
"cuda = torch.cuda.is_available()",
|
|
1752
|
+
"payload.update({'cuda_device': torch.cuda.get_device_name(0), 'compute_capability': list(torch.cuda.get_device_capability(0)), 'total_memory_bytes': torch.cuda.get_device_properties(0).total_memory, 'bf16_supported': bool(torch.cuda.is_bf16_supported())} if cuda else {})",
|
|
1753
|
+
"print(json.dumps(payload))"
|
|
1754
|
+
].join("; ");
|
|
1755
|
+
}
|
|
1756
|
+
function parsePythonProbeJson(text) {
|
|
1757
|
+
const line = [...text.split(/\r?\n/)].reverse().find((item) => item.trim().startsWith("{"));
|
|
1758
|
+
if (!line) return void 0;
|
|
1759
|
+
try {
|
|
1760
|
+
const parsed = JSON.parse(line);
|
|
1761
|
+
if (parsed.python_ok !== true) return void 0;
|
|
1762
|
+
return {
|
|
1763
|
+
ok: true,
|
|
1764
|
+
message: line.trim(),
|
|
1765
|
+
torch: typeof parsed.torch === "string" ? parsed.torch : void 0,
|
|
1766
|
+
transformers: typeof parsed.transformers === "string" ? parsed.transformers : void 0,
|
|
1767
|
+
peft: typeof parsed.peft === "string" ? parsed.peft : void 0,
|
|
1768
|
+
cuda_available: parsed.cuda_available === true,
|
|
1769
|
+
cuda_device: typeof parsed.cuda_device === "string" ? parsed.cuda_device : void 0,
|
|
1770
|
+
compute_capability: Array.isArray(parsed.compute_capability) ? parsed.compute_capability.filter((value) => typeof value === "number") : void 0,
|
|
1771
|
+
total_memory_bytes: typeof parsed.total_memory_bytes === "number" ? parsed.total_memory_bytes : void 0,
|
|
1772
|
+
bf16_supported: parsed.bf16_supported === true
|
|
1773
|
+
};
|
|
1774
|
+
} catch {
|
|
1775
|
+
return void 0;
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
async function collectPythonRuntime(config, env) {
|
|
1779
|
+
const entrypoint = buildBundledPythonCommand("-c", [pythonProbeSource()]);
|
|
1780
|
+
const childEnv = withBundledPythonEnvironment(
|
|
1781
|
+
withHuggingFaceCacheEnvironment(
|
|
1782
|
+
minimalMachineLearningEnvironment(env),
|
|
1783
|
+
config?.paths.modelCache
|
|
1784
|
+
)
|
|
1785
|
+
);
|
|
1786
|
+
const result = await runCommand(entrypoint.command, entrypoint.commandArgs, {
|
|
1787
|
+
env: childEnv,
|
|
1788
|
+
timeoutMs: 18e5
|
|
1789
|
+
});
|
|
1790
|
+
if (result.code === 0) {
|
|
1791
|
+
return parsePythonProbeJson(result.stdout) ?? {
|
|
1792
|
+
ok: true,
|
|
1793
|
+
message: firstLine(result.stdout) || "Python runtime probe succeeded"
|
|
1794
|
+
};
|
|
1795
|
+
}
|
|
1796
|
+
return {
|
|
1797
|
+
ok: false,
|
|
1798
|
+
message: result.error ?? (firstLine(result.stderr) || `uv exited ${result.code}`)
|
|
1799
|
+
};
|
|
1800
|
+
}
|
|
1801
|
+
async function collectHostInventory(options = {}) {
|
|
1802
|
+
const env = options.env ?? process.env;
|
|
1803
|
+
const config = options.config;
|
|
1804
|
+
const cwd = options.cwd ?? process.cwd();
|
|
1805
|
+
const nodeVersion = process.versions.node;
|
|
1806
|
+
const nodeMajor = Number(nodeVersion.split(".")[0]);
|
|
1807
|
+
const artifactRoot = resolve7(cwd, config?.artifactRoot ?? join5(cwd, DEFAULT_ARTIFACT_ROOT));
|
|
1808
|
+
const storeRoot = resolve7(config?.storeRoot ?? defaultStoreRoot(env));
|
|
1809
|
+
const modelCache = resolve7(
|
|
1810
|
+
config?.paths.modelCache ?? env.HF_HOME ?? join5(homedir3(), ".cache", "huggingface")
|
|
1811
|
+
);
|
|
1812
|
+
const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
|
|
1813
|
+
diskCheck("artifact-root", artifactRoot),
|
|
1814
|
+
diskCheck("store-root", storeRoot),
|
|
1815
|
+
diskCheck("model-cache", modelCache),
|
|
1816
|
+
runCommand("uv", ["--version"], { env, timeoutMs: 1e4 }),
|
|
1817
|
+
queryNvidiaGpus(env)
|
|
1818
|
+
]);
|
|
1819
|
+
const nvidia = nvidiaQueried.result;
|
|
1820
|
+
const gpus = nvidiaQueried.gpus;
|
|
1821
|
+
const inventory = {
|
|
1822
|
+
collected_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1823
|
+
quick: options.quick === true,
|
|
1824
|
+
node: {
|
|
1825
|
+
version: nodeVersion,
|
|
1826
|
+
major: nodeMajor,
|
|
1827
|
+
ok: nodeMajor >= 22
|
|
1828
|
+
},
|
|
1829
|
+
os: {
|
|
1830
|
+
platform: process.platform,
|
|
1831
|
+
arch: process.arch,
|
|
1832
|
+
type: type(),
|
|
1833
|
+
cpu_count: cpus().length,
|
|
1834
|
+
total_memory_bytes: totalmem(),
|
|
1835
|
+
free_memory_bytes: freemem()
|
|
1836
|
+
},
|
|
1837
|
+
uv: {
|
|
1838
|
+
ok: uvVersion.code === 0,
|
|
1839
|
+
version: uvVersion.code === 0 ? firstLine(uvVersion.stdout) : void 0,
|
|
1840
|
+
message: uvVersion.code === 0 ? firstLine(uvVersion.stdout) : uvVersion.error ?? (firstLine(uvVersion.stderr) || "uv is not available")
|
|
1841
|
+
},
|
|
1842
|
+
gpus,
|
|
1843
|
+
nvidia_smi: {
|
|
1844
|
+
ok: nvidia.code === 0,
|
|
1845
|
+
message: nvidia.code === 0 ? gpus[0] ? `${gpus[0].name}${gpus[0].memory_total_bytes ? ` (${Math.round(gpus[0].memory_total_bytes / 1024 ** 3)} GiB)` : ""}` : firstLine(nvidia.stdout) || "nvidia-smi reported a GPU" : nvidia.error ?? (firstLine(nvidia.stderr) || "nvidia-smi not available")
|
|
1846
|
+
},
|
|
1847
|
+
disks: [artifactDisk, storeDisk, cacheDisk]
|
|
1848
|
+
};
|
|
1849
|
+
if (options.quick !== true) {
|
|
1850
|
+
inventory.python = uvVersion.code === 0 ? await collectPythonRuntime(config, env) : { ok: false, message: "Python probe skipped because uv is not available" };
|
|
1851
|
+
}
|
|
1852
|
+
return inventory;
|
|
1853
|
+
}
|
|
1854
|
+
|
|
1855
|
+
// src/local-runtime/hardware-snapshot.ts
|
|
1856
|
+
import { chmod, mkdir as mkdir3, readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
|
|
1857
|
+
import { dirname as dirname6, join as join6 } from "path";
|
|
1858
|
+
var HARDWARE_SNAPSHOT_VERSION = 1;
|
|
1859
|
+
var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
1860
|
+
function hardwareSnapshotPath(env = process.env) {
|
|
1861
|
+
return join6(getTunedTensorHome(env), "hardware.json");
|
|
1862
|
+
}
|
|
1863
|
+
async function readHardwareSnapshot(env = process.env) {
|
|
1864
|
+
const path = hardwareSnapshotPath(env);
|
|
1865
|
+
try {
|
|
1866
|
+
const parsed = JSON.parse(await readFile2(path, "utf8"));
|
|
1867
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return void 0;
|
|
1868
|
+
const snapshot = parsed;
|
|
1869
|
+
if (snapshot.version !== HARDWARE_SNAPSHOT_VERSION) return void 0;
|
|
1870
|
+
if (typeof snapshot.collected_at !== "string" || !snapshot.capabilities || !snapshot.inventory) {
|
|
1871
|
+
return void 0;
|
|
1872
|
+
}
|
|
1873
|
+
const age = Date.now() - Date.parse(snapshot.collected_at);
|
|
1874
|
+
const stale = !Number.isFinite(age) || age > HARDWARE_SNAPSHOT_TTL_MS || age < 0;
|
|
1875
|
+
return { ...snapshot, stale, path };
|
|
1876
|
+
} catch (error) {
|
|
1877
|
+
if (error.code === "ENOENT") return void 0;
|
|
1878
|
+
return void 0;
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
async function writeHardwareSnapshot(snapshot, env = process.env) {
|
|
1882
|
+
const path = hardwareSnapshotPath(env);
|
|
1883
|
+
await mkdir3(dirname6(path), { recursive: true, mode: 448 });
|
|
1884
|
+
const gpu = snapshot.capabilities.gpu;
|
|
1885
|
+
const record = {
|
|
1886
|
+
version: HARDWARE_SNAPSHOT_VERSION,
|
|
1887
|
+
collected_at: snapshot.collected_at,
|
|
1888
|
+
quick: snapshot.quick,
|
|
1889
|
+
gpu_fingerprint: gpu ? { name: gpu.name, total_memory_bytes: gpu.memory_total_bytes } : void 0,
|
|
1890
|
+
inventory: snapshot.inventory,
|
|
1891
|
+
capabilities: snapshot.capabilities,
|
|
1892
|
+
summary: snapshot.summary ?? formatCapabilitySummary(snapshot.capabilities)
|
|
1893
|
+
};
|
|
1894
|
+
await writeFile2(path, `${JSON.stringify(record, null, 2)}
|
|
1895
|
+
`, { mode: 384 });
|
|
1896
|
+
await chmod(path, 384);
|
|
1897
|
+
return record;
|
|
1898
|
+
}
|
|
1899
|
+
function formatHostStatusLine(snapshot) {
|
|
1900
|
+
if (!snapshot) return "not inventoried (run tt hardware)";
|
|
1901
|
+
const stale = snapshot.stale ? `; stale \u2014 run tt hardware` : "";
|
|
1902
|
+
return `${snapshot.summary}${stale}`;
|
|
1903
|
+
}
|
|
1904
|
+
function formatAgentHostBlock(snapshot) {
|
|
1905
|
+
if (!snapshot) {
|
|
1906
|
+
return [
|
|
1907
|
+
"Host capability is not inventoried yet.",
|
|
1908
|
+
"When the user asks to examine this machine, GPU, VRAM, CUDA, or what can train, fine-tune, or infer here, call examine_hardware before recommending a base model, engine, or pipeline."
|
|
1909
|
+
].join(" ");
|
|
1910
|
+
}
|
|
1911
|
+
const stale = snapshot.stale ? " (stale \u2014 call examine_hardware to refresh)" : "";
|
|
1912
|
+
return [
|
|
1913
|
+
`Host capability (cached ${snapshot.collected_at}${stale}):`,
|
|
1914
|
+
snapshot.summary,
|
|
1915
|
+
"When the user wants to examine this host, GPU, or decide what can train/fine-tune/infer here, call examine_hardware before recommending a base model, engine, or pipeline.",
|
|
1916
|
+
"Recommend only workloads marked ready; mention tight as a caution. Never invent generic 7B/70B sizing."
|
|
1917
|
+
].join("\n");
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
// src/local-runtime/hardware.ts
|
|
1921
|
+
async function assessHardware(options = {}) {
|
|
1922
|
+
const env = options.env ?? process.env;
|
|
1923
|
+
if (options.quick === true) {
|
|
1924
|
+
const existing = await readHardwareSnapshot(env);
|
|
1925
|
+
if (existing && !existing.quick && !existing.stale) {
|
|
1926
|
+
return {
|
|
1927
|
+
version: existing.version,
|
|
1928
|
+
collected_at: existing.collected_at,
|
|
1929
|
+
quick: existing.quick,
|
|
1930
|
+
gpu_fingerprint: existing.gpu_fingerprint,
|
|
1931
|
+
inventory: existing.inventory,
|
|
1932
|
+
capabilities: existing.capabilities,
|
|
1933
|
+
summary: existing.summary
|
|
1934
|
+
};
|
|
1935
|
+
}
|
|
1936
|
+
}
|
|
1937
|
+
const inventory = await collectHostInventory({
|
|
1938
|
+
config: options.config,
|
|
1939
|
+
quick: options.quick === true,
|
|
1940
|
+
env,
|
|
1941
|
+
cwd: options.cwd
|
|
1942
|
+
});
|
|
1943
|
+
const capabilities = evaluateCapabilities(inventory);
|
|
1944
|
+
return await writeHardwareSnapshot({
|
|
1945
|
+
collected_at: inventory.collected_at,
|
|
1946
|
+
quick: inventory.quick,
|
|
1947
|
+
inventory,
|
|
1948
|
+
capabilities,
|
|
1949
|
+
summary: formatCapabilitySummary(capabilities)
|
|
1950
|
+
}, env);
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1953
|
+
// src/local-runtime/store.ts
|
|
1954
|
+
import { appendFile, chmod as chmod2, copyFile as copyFile2, lstat as lstat3, mkdir as mkdir4, readdir as readdir3, readFile as readFile3, rename as rename2, rm, stat as stat2, writeFile as writeFile3 } from "fs/promises";
|
|
1955
|
+
import { dirname as dirname7, join as join7, resolve as resolve8, sep as sep2 } from "path";
|
|
1956
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
1957
|
+
function isTerminalRunState(state) {
|
|
1958
|
+
return state.status === "stage_completed" || state.status === "completed" || state.status === "failed" || state.status === "cancelled" && state.current_stage === "cancelled" && Boolean(state.completed_at);
|
|
1959
|
+
}
|
|
1960
|
+
function defaultLocalHome() {
|
|
1961
|
+
return defaultStoreRoot();
|
|
1962
|
+
}
|
|
1963
|
+
function localStorePaths(root) {
|
|
1964
|
+
return {
|
|
1965
|
+
root,
|
|
1966
|
+
specsDir: join7(root, "specs"),
|
|
1967
|
+
runsDir: join7(root, "runs"),
|
|
1968
|
+
modelsDir: join7(root, "models"),
|
|
1969
|
+
datasetsDir: join7(root, "datasets")
|
|
1970
|
+
};
|
|
1971
|
+
}
|
|
1972
|
+
async function exists(path) {
|
|
1973
|
+
try {
|
|
1974
|
+
await stat2(path);
|
|
1975
|
+
return true;
|
|
1976
|
+
} catch {
|
|
1977
|
+
return false;
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
async function ensurePrivateDirectory(path) {
|
|
1981
|
+
await mkdir4(path, { recursive: true, mode: 448 });
|
|
1982
|
+
const metadata = await lstat3(path);
|
|
1983
|
+
if (metadata.isSymbolicLink()) {
|
|
1984
|
+
throw new Error(`Refusing symbolic link in local store: ${path}`);
|
|
1985
|
+
}
|
|
1986
|
+
if (!metadata.isDirectory()) {
|
|
1987
|
+
throw new Error(`Expected local store directory: ${path}`);
|
|
1988
|
+
}
|
|
1989
|
+
await chmod2(path, 448);
|
|
1990
|
+
}
|
|
1991
|
+
async function hardenPrivateTree(path) {
|
|
1992
|
+
const metadata = await lstat3(path);
|
|
1993
|
+
if (metadata.isSymbolicLink()) {
|
|
1994
|
+
throw new Error(`Refusing symbolic link in local store: ${path}`);
|
|
1995
|
+
}
|
|
1996
|
+
if (metadata.isDirectory()) {
|
|
1997
|
+
await chmod2(path, 448);
|
|
1998
|
+
for (const entry of await readdir3(path)) {
|
|
1999
|
+
await hardenPrivateTree(join7(path, entry));
|
|
2000
|
+
}
|
|
2001
|
+
return;
|
|
2002
|
+
}
|
|
2003
|
+
if (metadata.isFile()) {
|
|
2004
|
+
await chmod2(path, 384);
|
|
2005
|
+
return;
|
|
2006
|
+
}
|
|
2007
|
+
throw new Error(`Refusing unsupported filesystem entry in local store: ${path}`);
|
|
2008
|
+
}
|
|
2009
|
+
async function hardenExistingLocalStore(root) {
|
|
2010
|
+
const resolvedRoot = resolve8(root);
|
|
2011
|
+
let metadata;
|
|
2012
|
+
try {
|
|
2013
|
+
metadata = await lstat3(resolvedRoot);
|
|
2014
|
+
} catch (error) {
|
|
2015
|
+
if (error.code === "ENOENT") return false;
|
|
2016
|
+
throw error;
|
|
2017
|
+
}
|
|
2018
|
+
if (metadata.isSymbolicLink()) {
|
|
2019
|
+
throw new Error(`Refusing symbolic link in local store: ${resolvedRoot}`);
|
|
2020
|
+
}
|
|
2021
|
+
if (!metadata.isDirectory()) {
|
|
2022
|
+
throw new Error(`Expected local store directory: ${resolvedRoot}`);
|
|
2023
|
+
}
|
|
2024
|
+
await hardenPrivateTree(resolvedRoot);
|
|
2025
|
+
return true;
|
|
2026
|
+
}
|
|
2027
|
+
async function writePrivateJsonAtomic(path, value) {
|
|
2028
|
+
await ensurePrivateDirectory(dirname7(path));
|
|
2029
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.${randomUUID2()}.tmp`;
|
|
2030
|
+
await writeFile3(tmp, `${JSON.stringify(value, null, 2)}
|
|
2031
|
+
`, {
|
|
2032
|
+
encoding: "utf8",
|
|
2033
|
+
mode: 384
|
|
2034
|
+
});
|
|
2035
|
+
await rename2(tmp, path);
|
|
2036
|
+
await chmod2(path, 384);
|
|
2037
|
+
}
|
|
2038
|
+
async function readJson2(path) {
|
|
2039
|
+
return JSON.parse(await readFile3(path, "utf8"));
|
|
2040
|
+
}
|
|
2041
|
+
async function appendJsonl(path, value) {
|
|
2042
|
+
await ensurePrivateDirectory(dirname7(path));
|
|
2043
|
+
await appendFile(path, `${JSON.stringify(value)}
|
|
2044
|
+
`, {
|
|
2045
|
+
encoding: "utf8",
|
|
2046
|
+
mode: 384
|
|
2047
|
+
});
|
|
2048
|
+
await chmod2(path, 384);
|
|
2049
|
+
}
|
|
2050
|
+
async function writePrivateText(path, value) {
|
|
2051
|
+
await ensurePrivateDirectory(dirname7(path));
|
|
2052
|
+
await writeFile3(path, value, { encoding: "utf8", mode: 384 });
|
|
2053
|
+
await chmod2(path, 384);
|
|
2054
|
+
}
|
|
2055
|
+
async function readJsonl(path) {
|
|
2056
|
+
if (!await exists(path)) return [];
|
|
2057
|
+
return (await readFile3(path, "utf8")).split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
|
|
2058
|
+
}
|
|
2059
|
+
async function copyIfExists(from, to) {
|
|
2060
|
+
if (!await exists(from)) return;
|
|
2061
|
+
await ensurePrivateDirectory(dirname7(to));
|
|
2062
|
+
await copyFile2(from, to);
|
|
2063
|
+
await chmod2(to, 384);
|
|
2064
|
+
}
|
|
2065
|
+
async function childDirectoryNames(path) {
|
|
2066
|
+
try {
|
|
2067
|
+
const entries = await readdir3(path, { withFileTypes: true });
|
|
2068
|
+
return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
2069
|
+
} catch (error) {
|
|
2070
|
+
if (error.code === "ENOENT") return [];
|
|
2071
|
+
throw error;
|
|
2072
|
+
}
|
|
2073
|
+
}
|
|
2074
|
+
function compareIds(left, right) {
|
|
2075
|
+
if (left.id === right.id) return 0;
|
|
2076
|
+
return left.id < right.id ? -1 : 1;
|
|
2077
|
+
}
|
|
2078
|
+
function compareUpdatedAt(left, right) {
|
|
2079
|
+
if (left.updated_at !== right.updated_at) return left.updated_at > right.updated_at ? -1 : 1;
|
|
2080
|
+
return compareIds(left, right);
|
|
2081
|
+
}
|
|
2082
|
+
function compareCreatedAt(left, right) {
|
|
2083
|
+
if (left.created_at !== right.created_at) return left.created_at > right.created_at ? -1 : 1;
|
|
2084
|
+
return compareIds(left, right);
|
|
2085
|
+
}
|
|
2086
|
+
function findByIdOrPrefix(records, id) {
|
|
2087
|
+
return records.find((record) => record.id === id) ?? records.find((record) => record.id.startsWith(id));
|
|
2088
|
+
}
|
|
2089
|
+
function isPathInside(root, candidate) {
|
|
2090
|
+
const resolvedRoot = resolve8(root);
|
|
2091
|
+
const resolvedCandidate = resolve8(candidate);
|
|
2092
|
+
return resolvedCandidate === resolvedRoot || resolvedCandidate.startsWith(`${resolvedRoot}${sep2}`);
|
|
2093
|
+
}
|
|
2094
|
+
function createLocalStore(root = defaultLocalHome()) {
|
|
2095
|
+
const resolvedRoot = resolve8(root);
|
|
2096
|
+
const paths = localStorePaths(resolvedRoot);
|
|
2097
|
+
const runDir = (id) => join7(paths.runsDir, id);
|
|
2098
|
+
const runStatePath = (id) => join7(runDir(id), "state.json");
|
|
2099
|
+
const runEventsPath = (id) => join7(runDir(id), "progress.jsonl");
|
|
2100
|
+
const runRequestPath = (id) => join7(runDir(id), "request.json");
|
|
2101
|
+
const runReportPath = (id) => join7(runDir(id), "run-report.json");
|
|
2102
|
+
const cancellationPath = (id) => join7(runDir(id), "cancel.requested");
|
|
2103
|
+
const specDir = (id) => join7(paths.specsDir, id);
|
|
2104
|
+
const specPath = (id) => join7(specDir(id), "spec.json");
|
|
2105
|
+
const modelPath = (id) => join7(paths.modelsDir, id, "model.json");
|
|
2106
|
+
let ensurePromise;
|
|
2107
|
+
async function ensureOnce() {
|
|
1058
2108
|
await ensurePrivateDirectory(paths.root);
|
|
1059
2109
|
await Promise.all([
|
|
1060
2110
|
ensurePrivateDirectory(paths.specsDir),
|
|
@@ -1136,7 +2186,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1136
2186
|
...event
|
|
1137
2187
|
};
|
|
1138
2188
|
await appendJsonl(runEventsPath(state.id), row);
|
|
1139
|
-
await appendJsonl(
|
|
2189
|
+
await appendJsonl(join7(state.artifact_dir, "progress.jsonl"), {
|
|
1140
2190
|
at: row.occurred_at,
|
|
1141
2191
|
stage: row.stage,
|
|
1142
2192
|
status: row.status,
|
|
@@ -1167,7 +2217,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1167
2217
|
const previous = await getRun(runId);
|
|
1168
2218
|
if (isTerminalRunState(previous)) return previous;
|
|
1169
2219
|
await Promise.all([
|
|
1170
|
-
rm(
|
|
2220
|
+
rm(join7(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true }),
|
|
1171
2221
|
rm(runReportPath(previous.id), { force: true })
|
|
1172
2222
|
]);
|
|
1173
2223
|
if (previous.status === "cancelled" && previous.current_stage === "cancel_requested") return previous;
|
|
@@ -1320,6 +2370,13 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1320
2370
|
}
|
|
1321
2371
|
const request = await readJson2(runRequestPath(report.run_id));
|
|
1322
2372
|
const completedAt = report.created_at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
2373
|
+
if (!await exists(reportPath)) {
|
|
2374
|
+
throw new Error(`Run report not found: ${reportPath}`);
|
|
2375
|
+
}
|
|
2376
|
+
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
2377
|
+
if (await exists(cancellationPath(report.run_id))) {
|
|
2378
|
+
return preserveCancellationRequestState(report.run_id);
|
|
2379
|
+
}
|
|
1323
2380
|
const model = await registerModelRecord({
|
|
1324
2381
|
request,
|
|
1325
2382
|
training: report.training,
|
|
@@ -1327,7 +2384,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1327
2384
|
createdAt: completedAt
|
|
1328
2385
|
});
|
|
1329
2386
|
if (!model) {
|
|
1330
|
-
await rm(
|
|
2387
|
+
await rm(join7(paths.modelsDir, `local-${report.run_id}`), { recursive: true, force: true });
|
|
1331
2388
|
}
|
|
1332
2389
|
const previous = await getRun(report.run_id);
|
|
1333
2390
|
const state = {
|
|
@@ -1341,10 +2398,6 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1341
2398
|
completed_at: completedAt,
|
|
1342
2399
|
updated_at: completedAt
|
|
1343
2400
|
};
|
|
1344
|
-
if (!await exists(reportPath)) {
|
|
1345
|
-
throw new Error(`Run report not found: ${reportPath}`);
|
|
1346
|
-
}
|
|
1347
|
-
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
1348
2401
|
if (await exists(cancellationPath(report.run_id))) {
|
|
1349
2402
|
return preserveCancellationRequestState(report.run_id);
|
|
1350
2403
|
}
|
|
@@ -1365,7 +2418,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1365
2418
|
const previous = await getRun(runId);
|
|
1366
2419
|
if (options.report) await rm(runReportPath(previous.id), { force: true });
|
|
1367
2420
|
if (options.model) {
|
|
1368
|
-
await rm(
|
|
2421
|
+
await rm(join7(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true });
|
|
1369
2422
|
}
|
|
1370
2423
|
const state = {
|
|
1371
2424
|
...previous,
|
|
@@ -1442,9 +2495,9 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1442
2495
|
const state = await getRun(id);
|
|
1443
2496
|
const candidates = [];
|
|
1444
2497
|
if (state.report_path) {
|
|
1445
|
-
const reportPath =
|
|
2498
|
+
const reportPath = resolve8(state.report_path);
|
|
1446
2499
|
const allowedRoots = [resolvedRoot];
|
|
1447
|
-
if (state.artifact_dir) allowedRoots.push(
|
|
2500
|
+
if (state.artifact_dir) allowedRoots.push(resolve8(state.artifact_dir));
|
|
1448
2501
|
if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) {
|
|
1449
2502
|
candidates.push(reportPath);
|
|
1450
2503
|
}
|
|
@@ -1744,23 +2797,23 @@ function isFoundationSpecFile(spec) {
|
|
|
1744
2797
|
}
|
|
1745
2798
|
|
|
1746
2799
|
// src/local-runtime/local-project.ts
|
|
1747
|
-
import { createHash as
|
|
1748
|
-
import { readFile as
|
|
1749
|
-
import { dirname as
|
|
1750
|
-
import { homedir as
|
|
1751
|
-
import { mkdir as
|
|
2800
|
+
import { createHash as createHash3, randomUUID as randomUUID3 } from "crypto";
|
|
2801
|
+
import { readFile as readFile4, stat as stat3, writeFile as writeFile4 } from "fs/promises";
|
|
2802
|
+
import { dirname as dirname8, isAbsolute as isAbsolute3, resolve as resolve9 } from "path";
|
|
2803
|
+
import { homedir as homedir4 } from "os";
|
|
2804
|
+
import { mkdir as mkdir5 } from "fs/promises";
|
|
1752
2805
|
var DEFAULT_LOCAL_SPEC_PATH = "tunedtensor.json";
|
|
1753
2806
|
function resolveLocalReference(value, baseDirectory) {
|
|
1754
2807
|
if (typeof value !== "string" || !value || isAbsolute3(value) || /^[a-z][a-z0-9+.-]*:/i.test(value)) {
|
|
1755
2808
|
return value;
|
|
1756
2809
|
}
|
|
1757
|
-
if (value === "~") return
|
|
1758
|
-
if (value.startsWith("~/")) return
|
|
1759
|
-
return
|
|
2810
|
+
if (value === "~") return homedir4();
|
|
2811
|
+
if (value.startsWith("~/")) return resolve9(homedir4(), value.slice(2));
|
|
2812
|
+
return resolve9(baseDirectory, value);
|
|
1760
2813
|
}
|
|
1761
2814
|
function resolveLocalRunInputPaths(raw, inputPath) {
|
|
1762
2815
|
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw;
|
|
1763
|
-
const baseDirectory =
|
|
2816
|
+
const baseDirectory = dirname8(resolve9(inputPath));
|
|
1764
2817
|
const value = structuredClone(raw);
|
|
1765
2818
|
const dataset = value.dataset_prebuilt;
|
|
1766
2819
|
if (dataset && typeof dataset === "object" && !Array.isArray(dataset)) {
|
|
@@ -1816,8 +2869,8 @@ async function initLocalSpecFile(args) {
|
|
|
1816
2869
|
nproc_per_node: 1
|
|
1817
2870
|
}
|
|
1818
2871
|
});
|
|
1819
|
-
await
|
|
1820
|
-
await
|
|
2872
|
+
await mkdir5(dirname8(args.outputPath), { recursive: true });
|
|
2873
|
+
await writeFile4(args.outputPath, `${JSON.stringify(spec2, null, 2)}
|
|
1821
2874
|
`, "utf8");
|
|
1822
2875
|
return spec2;
|
|
1823
2876
|
}
|
|
@@ -1848,8 +2901,8 @@ async function initLocalSpecFile(args) {
|
|
|
1848
2901
|
n_epochs: 1
|
|
1849
2902
|
}
|
|
1850
2903
|
});
|
|
1851
|
-
await
|
|
1852
|
-
await
|
|
2904
|
+
await mkdir5(dirname8(args.outputPath), { recursive: true });
|
|
2905
|
+
await writeFile4(args.outputPath, `${JSON.stringify(spec, null, 2)}
|
|
1853
2906
|
`, "utf8");
|
|
1854
2907
|
return spec;
|
|
1855
2908
|
}
|
|
@@ -1870,8 +2923,8 @@ async function initLocalRunnerConfigFile(args) {
|
|
|
1870
2923
|
timeoutMs: 18e5
|
|
1871
2924
|
}
|
|
1872
2925
|
};
|
|
1873
|
-
await
|
|
1874
|
-
await
|
|
2926
|
+
await mkdir5(dirname8(args.outputPath), { recursive: true });
|
|
2927
|
+
await writeFile4(args.outputPath, `${JSON.stringify(config, null, 2)}
|
|
1875
2928
|
`, "utf8");
|
|
1876
2929
|
return config;
|
|
1877
2930
|
}
|
|
@@ -1907,536 +2960,185 @@ function assertFoundationSpecReady(spec) {
|
|
|
1907
2960
|
}
|
|
1908
2961
|
function runRequestFromLocalSpec(spec, options = {}) {
|
|
1909
2962
|
if (isFoundationSpecFile(spec)) {
|
|
1910
|
-
throw new Error(
|
|
1911
|
-
"Foundation specs cannot be converted into a LoRA run request. Use `tt pipeline run --spec`."
|
|
1912
|
-
);
|
|
1913
|
-
}
|
|
1914
|
-
const {
|
|
1915
|
-
id,
|
|
1916
|
-
engine: _engine,
|
|
1917
|
-
hyperparameters,
|
|
1918
|
-
dataset_prebuilt: datasetPrebuilt,
|
|
1919
|
-
...specSnapshot
|
|
1920
|
-
} = spec;
|
|
1921
|
-
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
1922
|
-
return fineTuneRunRequestSchema.parse({
|
|
1923
|
-
run_id: options.runId ?? randomUUID3(),
|
|
1924
|
-
user_id: "local-user",
|
|
1925
|
-
behavior_spec_id: behaviorSpecId,
|
|
1926
|
-
run_number: 1,
|
|
1927
|
-
spec_snapshot: specSnapshot,
|
|
1928
|
-
hyperparameters,
|
|
1929
|
-
dataset_prebuilt: datasetPrebuilt
|
|
1930
|
-
});
|
|
1931
|
-
}
|
|
1932
|
-
function deterministicBehaviorSpecId(spec) {
|
|
1933
|
-
const namespace = Buffer.from("9f783f01340457bea061942004259253", "hex");
|
|
1934
|
-
const digest = createHash2("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
|
|
1935
|
-
digest[6] = digest[6] & 15 | 80;
|
|
1936
|
-
digest[8] = digest[8] & 63 | 128;
|
|
1937
|
-
const hex = digest.toString("hex");
|
|
1938
|
-
return [
|
|
1939
|
-
hex.slice(0, 8),
|
|
1940
|
-
hex.slice(8, 12),
|
|
1941
|
-
hex.slice(12, 16),
|
|
1942
|
-
hex.slice(16, 20),
|
|
1943
|
-
hex.slice(20)
|
|
1944
|
-
].join("-");
|
|
1945
|
-
}
|
|
1946
|
-
async function loadLocalRunInput(path, options = {}) {
|
|
1947
|
-
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile3(path, "utf8")), path);
|
|
1948
|
-
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
1949
|
-
if (request.success) {
|
|
1950
|
-
return { kind: "request", path, request: request.data };
|
|
1951
|
-
}
|
|
1952
|
-
const spec = localBehaviorSpecFileSchema.parse(raw);
|
|
1953
|
-
if (isFoundationSpecFile(spec)) {
|
|
1954
|
-
return { kind: "foundation-spec", path, spec };
|
|
1955
|
-
}
|
|
1956
|
-
return {
|
|
1957
|
-
kind: "spec",
|
|
1958
|
-
path,
|
|
1959
|
-
spec,
|
|
1960
|
-
request: runRequestFromLocalSpec(spec, options)
|
|
1961
|
-
};
|
|
1962
|
-
}
|
|
1963
|
-
|
|
1964
|
-
// src/local-runtime/dataset.ts
|
|
1965
|
-
import { readFile as readFile4 } from "fs/promises";
|
|
1966
|
-
async function loadNormalizedChatJsonl(path) {
|
|
1967
|
-
const text = await readFile4(path, "utf8");
|
|
1968
|
-
const rows = [];
|
|
1969
|
-
const jsonLines = [];
|
|
1970
|
-
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
1971
|
-
if (!line.trim()) continue;
|
|
1972
|
-
let value;
|
|
1973
|
-
try {
|
|
1974
|
-
value = JSON.parse(line);
|
|
1975
|
-
} catch (error) {
|
|
1976
|
-
throw new Error(`Invalid chat JSONL row ${index + 1}: malformed JSON`, { cause: error });
|
|
1977
|
-
}
|
|
1978
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1979
|
-
throw new Error(`Invalid chat JSONL row ${index + 1}: expected an object`);
|
|
1980
|
-
}
|
|
1981
|
-
const messages = value.messages;
|
|
1982
|
-
if (!Array.isArray(messages) || ![2, 3].includes(messages.length) || messages.some(
|
|
1983
|
-
(message) => !message || typeof message !== "object" || Array.isArray(message) || !["system", "user", "assistant"].includes(String(message.role)) || typeof message.content !== "string"
|
|
1984
|
-
)) {
|
|
1985
|
-
throw new Error(
|
|
1986
|
-
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one user message, and one assistant answer`
|
|
1987
|
-
);
|
|
1988
|
-
}
|
|
1989
|
-
const row = { messages };
|
|
1990
|
-
const offset = row.messages.length === 3 ? 1 : 0;
|
|
1991
|
-
if (offset === 1 && row.messages[0]?.role !== "system" || row.messages[offset]?.role !== "user" || !row.messages[offset]?.content.trim() || row.messages[offset + 1]?.role !== "assistant" || !row.messages[offset + 1]?.content.trim()) {
|
|
1992
|
-
throw new Error(
|
|
1993
|
-
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one non-empty user message, and one non-empty assistant answer`
|
|
1994
|
-
);
|
|
1995
|
-
}
|
|
1996
|
-
rows.push(row);
|
|
1997
|
-
jsonLines.push(JSON.stringify(row));
|
|
1998
|
-
}
|
|
1999
|
-
if (rows.length === 0) throw new Error(`Chat JSONL contains no examples: ${path}`);
|
|
2000
|
-
return {
|
|
2001
|
-
rows,
|
|
2002
|
-
jsonl: jsonLines.join("\n")
|
|
2003
|
-
};
|
|
2004
|
-
}
|
|
2005
|
-
async function normalizeChatJsonlForRelocation(path) {
|
|
2006
|
-
return (await loadNormalizedChatJsonl(path)).jsonl;
|
|
2007
|
-
}
|
|
2008
|
-
function buildSystemMessage(spec) {
|
|
2009
|
-
const parts = [];
|
|
2010
|
-
if (spec.system_prompt.trim()) parts.push(spec.system_prompt.trim());
|
|
2011
|
-
if (spec.guidelines.length > 0) {
|
|
2012
|
-
parts.push(`Guidelines:
|
|
2013
|
-
${spec.guidelines.map((guideline) => `- ${guideline}`).join("\n")}`);
|
|
2014
|
-
}
|
|
2015
|
-
if (spec.constraints.length > 0) {
|
|
2016
|
-
parts.push(`Constraints:
|
|
2017
|
-
${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
|
|
2018
|
-
}
|
|
2019
|
-
return parts.join("\n\n") || "Follow the demonstrated behavior.";
|
|
2020
|
-
}
|
|
2021
|
-
function exampleToChatRow(spec, example) {
|
|
2022
|
-
const system = buildSystemMessage(spec);
|
|
2023
|
-
return {
|
|
2024
|
-
messages: [
|
|
2025
|
-
{ role: "system", content: system },
|
|
2026
|
-
{ role: "user", content: example.input },
|
|
2027
|
-
{ role: "assistant", content: example.output }
|
|
2028
|
-
]
|
|
2029
|
-
};
|
|
2030
|
-
}
|
|
2031
|
-
function compileSpecToJsonl(spec) {
|
|
2032
|
-
return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
|
|
2033
|
-
}
|
|
2034
|
-
function examplesFromSpec(spec) {
|
|
2035
|
-
return spec.examples;
|
|
2036
|
-
}
|
|
2037
|
-
async function examplesFromChatJsonl(path) {
|
|
2038
|
-
const normalized = await loadNormalizedChatJsonl(path);
|
|
2039
|
-
const examples = [];
|
|
2040
|
-
for (const [index, row] of normalized.rows.entries()) {
|
|
2041
|
-
const user = row.messages.at(-2);
|
|
2042
|
-
const assistant = row.messages.at(-1);
|
|
2043
|
-
examples.push({
|
|
2044
|
-
input: user.content,
|
|
2045
|
-
output: assistant.content
|
|
2046
|
-
});
|
|
2047
|
-
}
|
|
2048
|
-
return examples;
|
|
2049
|
-
}
|
|
2050
|
-
async function evaluationSuiteFromChatJsonl(path, configuredSystem) {
|
|
2051
|
-
const normalized = await loadNormalizedChatJsonl(path);
|
|
2052
|
-
const inputIdentities = /* @__PURE__ */ new Set();
|
|
2053
|
-
for (const row of normalized.rows) {
|
|
2054
|
-
const input = row.messages.at(-2).content;
|
|
2055
|
-
const identity = input.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2056
|
-
if (inputIdentities.has(identity)) {
|
|
2057
|
-
throw new Error(
|
|
2058
|
-
"General regression data contains duplicate inputs; evaluation prompts must be unique."
|
|
2059
|
-
);
|
|
2060
|
-
}
|
|
2061
|
-
inputIdentities.add(identity);
|
|
2062
|
-
}
|
|
2063
|
-
const systems = new Set(
|
|
2064
|
-
normalized.rows.map((row) => row.messages.find((message) => message.role === "system")?.content).filter((value) => value !== void 0)
|
|
2065
|
-
);
|
|
2066
|
-
if (systems.size > 1) {
|
|
2067
|
-
throw new Error("General regression rows must use one consistent system message.");
|
|
2068
|
-
}
|
|
2069
|
-
const embeddedSystem = [...systems][0];
|
|
2070
|
-
if (configuredSystem !== void 0 && embeddedSystem !== void 0 && configuredSystem !== embeddedSystem) {
|
|
2071
|
-
throw new Error(
|
|
2072
|
-
"evaluation.generalRegression.systemPrompt must match the system message embedded in the dataset."
|
|
2073
|
-
);
|
|
2074
|
-
}
|
|
2075
|
-
return {
|
|
2076
|
-
examples: normalized.rows.map((row) => ({
|
|
2077
|
-
input: row.messages.at(-2).content,
|
|
2078
|
-
output: row.messages.at(-1).content
|
|
2079
|
-
})),
|
|
2080
|
-
system: configuredSystem ?? embeddedSystem ?? "You are a helpful assistant."
|
|
2081
|
-
};
|
|
2082
|
-
}
|
|
2083
|
-
|
|
2084
|
-
// src/local-runtime/run-reporter.ts
|
|
2085
|
-
function reportInBackground(action) {
|
|
2086
|
-
try {
|
|
2087
|
-
void Promise.resolve(action()).catch(() => void 0);
|
|
2088
|
-
} catch {
|
|
2089
|
-
}
|
|
2090
|
-
}
|
|
2091
|
-
function sanitizeLogLine(line) {
|
|
2092
|
-
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
2093
|
-
}
|
|
2094
|
-
function forwardStreamLines(stream, onLine) {
|
|
2095
|
-
let buffered = "";
|
|
2096
|
-
stream.setEncoding("utf8");
|
|
2097
|
-
stream.on("data", (chunk) => {
|
|
2098
|
-
buffered += chunk;
|
|
2099
|
-
const lines = buffered.split(/\r\n|\n|\r/);
|
|
2100
|
-
buffered = lines.pop() ?? "";
|
|
2101
|
-
for (const line of lines) {
|
|
2102
|
-
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
2103
|
-
}
|
|
2104
|
-
});
|
|
2105
|
-
stream.on("end", () => {
|
|
2106
|
-
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
2107
|
-
});
|
|
2108
|
-
}
|
|
2109
|
-
|
|
2110
|
-
// src/local-runtime/package-root.ts
|
|
2111
|
-
import { existsSync as existsSync2 } from "fs";
|
|
2112
|
-
import { dirname as dirname4, resolve as resolve6 } from "path";
|
|
2113
|
-
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
2114
|
-
function localRuntimePackageRoot(moduleUrl) {
|
|
2115
|
-
const moduleDirectory = dirname4(fileURLToPath2(moduleUrl));
|
|
2116
|
-
const candidates = [
|
|
2117
|
-
dirname4(moduleDirectory),
|
|
2118
|
-
dirname4(dirname4(moduleDirectory))
|
|
2119
|
-
];
|
|
2120
|
-
const root = candidates.find(
|
|
2121
|
-
(candidate) => existsSync2(resolve6(candidate, "package.json")) && existsSync2(resolve6(candidate, "training/adapter"))
|
|
2122
|
-
);
|
|
2123
|
-
if (!root) {
|
|
2124
|
-
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
2125
|
-
}
|
|
2126
|
-
return root;
|
|
2127
|
-
}
|
|
2128
|
-
|
|
2129
|
-
// src/local-runtime/prefetch.ts
|
|
2130
|
-
import { createHash as createHash4 } from "crypto";
|
|
2131
|
-
import { createReadStream as createReadStream2 } from "fs";
|
|
2132
|
-
import { lstat as lstat4, mkdir as mkdir5, readFile as readFile5, readdir as readdir4, realpath as realpath2, stat as stat4 } from "fs/promises";
|
|
2133
|
-
import { isAbsolute as isAbsolute4, join as join6, relative as relative2, resolve as resolve9, sep as sep3 } from "path";
|
|
2134
|
-
|
|
2135
|
-
// src/local-runtime/process-runner.ts
|
|
2136
|
-
import { createHash as createHash3 } from "crypto";
|
|
2137
|
-
import { spawn } from "child_process";
|
|
2138
|
-
import { createWriteStream, readFileSync } from "fs";
|
|
2139
|
-
import { mkdir as mkdir4 } from "fs/promises";
|
|
2140
|
-
import { dirname as dirname5, join as join4, resolve as resolve7 } from "path";
|
|
2141
|
-
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
2142
|
-
var bundledProject = join4(packageRoot, "training/adapter");
|
|
2143
|
-
var bundledRuntimeHash = (() => {
|
|
2144
|
-
const hash = createHash3("sha256");
|
|
2145
|
-
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
2146
|
-
hash.update(readFileSync(join4(bundledProject, name)));
|
|
2147
|
-
}
|
|
2148
|
-
return hash.digest("hex").slice(0, 20);
|
|
2149
|
-
})();
|
|
2150
|
-
var BUNDLED_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv", bundledRuntimeHash);
|
|
2151
|
-
var foundationProject = join4(packageRoot, "training/foundation");
|
|
2152
|
-
var foundationRuntimeHash = (() => {
|
|
2153
|
-
const hash = createHash3("sha256");
|
|
2154
|
-
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
2155
|
-
hash.update(readFileSync(join4(foundationProject, name)));
|
|
2156
|
-
}
|
|
2157
|
-
return hash.digest("hex").slice(0, 20);
|
|
2158
|
-
})();
|
|
2159
|
-
var FOUNDATION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-foundation", foundationRuntimeHash);
|
|
2160
|
-
function buildBundledPythonCommand(entrypoint, args = []) {
|
|
2161
|
-
const target = entrypoint === "-c" ? entrypoint : join4(bundledProject, "src", entrypoint);
|
|
2162
|
-
const commandArgs = [
|
|
2163
|
-
"run",
|
|
2164
|
-
"--frozen",
|
|
2165
|
-
"--quiet",
|
|
2166
|
-
"--project",
|
|
2167
|
-
bundledProject,
|
|
2168
|
-
"python",
|
|
2169
|
-
target,
|
|
2170
|
-
...args
|
|
2171
|
-
];
|
|
2172
|
-
return {
|
|
2173
|
-
command: "uv",
|
|
2174
|
-
commandArgs,
|
|
2175
|
-
displayCommand: ["uv", ...commandArgs]
|
|
2176
|
-
};
|
|
2177
|
-
}
|
|
2178
|
-
function withBundledPythonEnvironment(env) {
|
|
2179
|
-
return {
|
|
2180
|
-
...env,
|
|
2181
|
-
UV_PROJECT_ENVIRONMENT: BUNDLED_PYTHON_ENVIRONMENT
|
|
2182
|
-
};
|
|
2183
|
-
}
|
|
2184
|
-
function buildFoundationPythonCommand(entrypoint, args = []) {
|
|
2185
|
-
const commandArgs = [
|
|
2186
|
-
"run",
|
|
2187
|
-
"--frozen",
|
|
2188
|
-
"--quiet",
|
|
2189
|
-
"--project",
|
|
2190
|
-
foundationProject,
|
|
2191
|
-
"python",
|
|
2192
|
-
join4(foundationProject, "src", entrypoint),
|
|
2193
|
-
...args
|
|
2194
|
-
];
|
|
2195
|
-
return {
|
|
2196
|
-
command: "uv",
|
|
2197
|
-
commandArgs,
|
|
2198
|
-
displayCommand: ["uv", ...commandArgs]
|
|
2199
|
-
};
|
|
2200
|
-
}
|
|
2201
|
-
function withFoundationPythonEnvironment(env) {
|
|
2202
|
-
return {
|
|
2203
|
-
...env,
|
|
2204
|
-
UV_PROJECT_ENVIRONMENT: FOUNDATION_PYTHON_ENVIRONMENT
|
|
2205
|
-
};
|
|
2206
|
-
}
|
|
2207
|
-
var ProcessCancelledError = class extends Error {
|
|
2208
|
-
constructor(message = "Process cancelled.") {
|
|
2209
|
-
super(message);
|
|
2210
|
-
this.name = "ProcessCancelledError";
|
|
2211
|
-
}
|
|
2212
|
-
};
|
|
2213
|
-
async function openProcessLog(path, exclusive) {
|
|
2214
|
-
const stream = createWriteStream(path, {
|
|
2215
|
-
flags: exclusive ? "wx" : "w",
|
|
2216
|
-
mode: 384
|
|
2217
|
-
});
|
|
2218
|
-
await new Promise((resolveOpen, reject) => {
|
|
2219
|
-
const onOpen = () => {
|
|
2220
|
-
stream.off("error", onError);
|
|
2221
|
-
resolveOpen();
|
|
2222
|
-
};
|
|
2223
|
-
const onError = (error) => {
|
|
2224
|
-
stream.off("open", onOpen);
|
|
2225
|
-
reject(error);
|
|
2226
|
-
};
|
|
2227
|
-
stream.once("open", onOpen);
|
|
2228
|
-
stream.once("error", onError);
|
|
2229
|
-
});
|
|
2230
|
-
return stream;
|
|
2231
|
-
}
|
|
2232
|
-
async function runLoggedProcess(args) {
|
|
2233
|
-
if (args.logPath) await mkdir4(dirname5(args.logPath), { recursive: true });
|
|
2234
|
-
const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false) : null;
|
|
2235
|
-
let stderr = "";
|
|
2236
|
-
try {
|
|
2237
|
-
const exitCode = await new Promise((resolvePromise, reject) => {
|
|
2238
|
-
const child = spawn(args.command, args.commandArgs, {
|
|
2239
|
-
cwd: args.cwd ? resolve7(args.cwd) : process.cwd(),
|
|
2240
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
2241
|
-
env: args.env ?? process.env,
|
|
2242
|
-
detached: process.platform !== "win32"
|
|
2243
|
-
});
|
|
2244
|
-
let timedOut = false;
|
|
2245
|
-
let cancelled = false;
|
|
2246
|
-
let cancellationError;
|
|
2247
|
-
let cancellationCheckRunning = false;
|
|
2248
|
-
let forceKillTimer = null;
|
|
2249
|
-
const killProcessGroup = (signal) => {
|
|
2250
|
-
if (child.pid && process.platform !== "win32") {
|
|
2251
|
-
try {
|
|
2252
|
-
process.kill(-child.pid, signal);
|
|
2253
|
-
return;
|
|
2254
|
-
} catch {
|
|
2255
|
-
}
|
|
2256
|
-
}
|
|
2257
|
-
child.kill(signal);
|
|
2258
|
-
};
|
|
2259
|
-
const requestStop = (signal = "SIGTERM") => {
|
|
2260
|
-
killProcessGroup(signal);
|
|
2261
|
-
if (!forceKillTimer) {
|
|
2262
|
-
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), 5e3);
|
|
2263
|
-
forceKillTimer.unref();
|
|
2264
|
-
}
|
|
2265
|
-
};
|
|
2266
|
-
const onSigint = () => {
|
|
2267
|
-
cancelled = true;
|
|
2268
|
-
requestStop("SIGINT");
|
|
2269
|
-
};
|
|
2270
|
-
const onSigterm = () => {
|
|
2271
|
-
cancelled = true;
|
|
2272
|
-
requestStop("SIGTERM");
|
|
2273
|
-
};
|
|
2274
|
-
process.once("SIGINT", onSigint);
|
|
2275
|
-
process.once("SIGTERM", onSigterm);
|
|
2276
|
-
const timer = args.timeoutMs ? setTimeout(() => {
|
|
2277
|
-
timedOut = true;
|
|
2278
|
-
requestStop();
|
|
2279
|
-
}, args.timeoutMs) : null;
|
|
2280
|
-
const cancellationTimer = args.shouldCancel ? setInterval(() => {
|
|
2281
|
-
if (cancellationCheckRunning || cancelled) return;
|
|
2282
|
-
cancellationCheckRunning = true;
|
|
2283
|
-
Promise.resolve(args.shouldCancel?.()).then((requested) => {
|
|
2284
|
-
if (!requested || cancelled) return;
|
|
2285
|
-
cancelled = true;
|
|
2286
|
-
requestStop();
|
|
2287
|
-
}).catch((error) => {
|
|
2288
|
-
cancellationError = error;
|
|
2289
|
-
requestStop();
|
|
2290
|
-
}).finally(() => {
|
|
2291
|
-
cancellationCheckRunning = false;
|
|
2292
|
-
});
|
|
2293
|
-
}, args.cancelPollMs ?? 250) : null;
|
|
2294
|
-
cancellationTimer?.unref();
|
|
2295
|
-
const clearProcessTimers = () => {
|
|
2296
|
-
if (timer) clearTimeout(timer);
|
|
2297
|
-
if (cancellationTimer) clearInterval(cancellationTimer);
|
|
2298
|
-
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
2299
|
-
process.off("SIGINT", onSigint);
|
|
2300
|
-
process.off("SIGTERM", onSigterm);
|
|
2301
|
-
};
|
|
2302
|
-
if (logStream) {
|
|
2303
|
-
child.stdout.pipe(logStream, { end: false });
|
|
2304
|
-
child.stderr.pipe(logStream, { end: false });
|
|
2305
|
-
}
|
|
2306
|
-
forwardStreamLines(child.stdout, (line) => {
|
|
2307
|
-
args.onLine?.(line, "stdout");
|
|
2308
|
-
if (args.reporter?.verbose) {
|
|
2309
|
-
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stdout", message: line }));
|
|
2310
|
-
}
|
|
2311
|
-
});
|
|
2312
|
-
forwardStreamLines(child.stderr, (line) => {
|
|
2313
|
-
stderr += `${line}
|
|
2314
|
-
`;
|
|
2315
|
-
args.onLine?.(line, "stderr");
|
|
2316
|
-
if (args.reporter?.verbose) {
|
|
2317
|
-
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stderr", message: line }));
|
|
2318
|
-
}
|
|
2319
|
-
});
|
|
2320
|
-
child.on("error", (error) => {
|
|
2321
|
-
clearProcessTimers();
|
|
2322
|
-
reject(timedOut ? new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`) : error);
|
|
2323
|
-
});
|
|
2324
|
-
child.on("close", (code) => {
|
|
2325
|
-
if (timedOut || cancelled || cancellationError || args.terminateProcessGroupOnExit) {
|
|
2326
|
-
killProcessGroup("SIGKILL");
|
|
2327
|
-
}
|
|
2328
|
-
clearProcessTimers();
|
|
2329
|
-
if (timedOut) {
|
|
2330
|
-
reject(new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`));
|
|
2331
|
-
return;
|
|
2332
|
-
}
|
|
2333
|
-
if (cancellationError) {
|
|
2334
|
-
reject(cancellationError);
|
|
2335
|
-
return;
|
|
2336
|
-
}
|
|
2337
|
-
if (cancelled) {
|
|
2338
|
-
reject(new ProcessCancelledError(`${args.stage} was cancelled.`));
|
|
2339
|
-
return;
|
|
2340
|
-
}
|
|
2341
|
-
resolvePromise(code ?? 1);
|
|
2342
|
-
});
|
|
2343
|
-
});
|
|
2344
|
-
return { exitCode, stderr };
|
|
2345
|
-
} finally {
|
|
2346
|
-
if (logStream) {
|
|
2347
|
-
await new Promise((resolveEnd) => {
|
|
2348
|
-
logStream.end(resolveEnd);
|
|
2349
|
-
});
|
|
2350
|
-
}
|
|
2963
|
+
throw new Error(
|
|
2964
|
+
"Foundation specs cannot be converted into a LoRA run request. Use `tt pipeline run --spec`."
|
|
2965
|
+
);
|
|
2351
2966
|
}
|
|
2967
|
+
const {
|
|
2968
|
+
id,
|
|
2969
|
+
engine: _engine,
|
|
2970
|
+
hyperparameters,
|
|
2971
|
+
dataset_prebuilt: datasetPrebuilt,
|
|
2972
|
+
...specSnapshot
|
|
2973
|
+
} = spec;
|
|
2974
|
+
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
2975
|
+
return fineTuneRunRequestSchema.parse({
|
|
2976
|
+
run_id: options.runId ?? randomUUID3(),
|
|
2977
|
+
user_id: "local-user",
|
|
2978
|
+
behavior_spec_id: behaviorSpecId,
|
|
2979
|
+
run_number: 1,
|
|
2980
|
+
spec_snapshot: specSnapshot,
|
|
2981
|
+
hyperparameters,
|
|
2982
|
+
dataset_prebuilt: datasetPrebuilt
|
|
2983
|
+
});
|
|
2352
2984
|
}
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
const
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
"TMP",
|
|
2367
|
-
"LANG",
|
|
2368
|
-
"LANGUAGE",
|
|
2369
|
-
"LD_LIBRARY_PATH",
|
|
2370
|
-
"DYLD_LIBRARY_PATH",
|
|
2371
|
-
"XDG_CACHE_HOME",
|
|
2372
|
-
"SSL_CERT_FILE",
|
|
2373
|
-
"REQUESTS_CA_BUNDLE",
|
|
2374
|
-
"CURL_CA_BUNDLE",
|
|
2375
|
-
"NO_PROXY",
|
|
2376
|
-
"no_proxy",
|
|
2377
|
-
"UV_CACHE_DIR",
|
|
2378
|
-
"UV_PYTHON",
|
|
2379
|
-
"HF_HOME",
|
|
2380
|
-
"HF_HUB_CACHE",
|
|
2381
|
-
"HUGGINGFACE_HUB_CACHE",
|
|
2382
|
-
"HF_ENDPOINT",
|
|
2383
|
-
"HF_TOKEN",
|
|
2384
|
-
"HF_HUB_OFFLINE",
|
|
2385
|
-
"HF_HUB_DISABLE_TELEMETRY",
|
|
2386
|
-
"TRANSFORMERS_OFFLINE",
|
|
2387
|
-
"HF_HUB_ENABLE_HF_TRANSFER",
|
|
2388
|
-
"PYTHONUTF8",
|
|
2389
|
-
"PYTHONIOENCODING"
|
|
2390
|
-
]);
|
|
2391
|
-
const prefixes = [
|
|
2392
|
-
"LC_",
|
|
2393
|
-
"CUDA_",
|
|
2394
|
-
"NVIDIA_",
|
|
2395
|
-
"NCCL_",
|
|
2396
|
-
"OMP_",
|
|
2397
|
-
"MKL_",
|
|
2398
|
-
"TORCH_",
|
|
2399
|
-
"PYTORCH_",
|
|
2400
|
-
"TOKENIZERS_",
|
|
2401
|
-
"HF_XET_"
|
|
2402
|
-
];
|
|
2403
|
-
const result = Object.fromEntries(Object.entries(env).filter(
|
|
2404
|
-
([key]) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix))
|
|
2405
|
-
));
|
|
2406
|
-
result.TORCH_DISABLE_NATIVE_JIT ??= "1";
|
|
2407
|
-
return result;
|
|
2985
|
+
function deterministicBehaviorSpecId(spec) {
|
|
2986
|
+
const namespace = Buffer.from("9f783f01340457bea061942004259253", "hex");
|
|
2987
|
+
const digest = createHash3("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
|
|
2988
|
+
digest[6] = digest[6] & 15 | 80;
|
|
2989
|
+
digest[8] = digest[8] & 63 | 128;
|
|
2990
|
+
const hex = digest.toString("hex");
|
|
2991
|
+
return [
|
|
2992
|
+
hex.slice(0, 8),
|
|
2993
|
+
hex.slice(8, 12),
|
|
2994
|
+
hex.slice(12, 16),
|
|
2995
|
+
hex.slice(16, 20),
|
|
2996
|
+
hex.slice(20)
|
|
2997
|
+
].join("-");
|
|
2408
2998
|
}
|
|
2409
|
-
function
|
|
2410
|
-
const
|
|
2999
|
+
async function loadLocalRunInput(path, options = {}) {
|
|
3000
|
+
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile4(path, "utf8")), path);
|
|
3001
|
+
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
3002
|
+
if (request.success) {
|
|
3003
|
+
return { kind: "request", path, request: request.data };
|
|
3004
|
+
}
|
|
3005
|
+
const spec = localBehaviorSpecFileSchema.parse(raw);
|
|
3006
|
+
if (isFoundationSpecFile(spec)) {
|
|
3007
|
+
return { kind: "foundation-spec", path, spec };
|
|
3008
|
+
}
|
|
2411
3009
|
return {
|
|
2412
|
-
|
|
2413
|
-
|
|
3010
|
+
kind: "spec",
|
|
3011
|
+
path,
|
|
3012
|
+
spec,
|
|
3013
|
+
request: runRequestFromLocalSpec(spec, options)
|
|
2414
3014
|
};
|
|
2415
3015
|
}
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
const
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
3016
|
+
|
|
3017
|
+
// src/local-runtime/dataset.ts
|
|
3018
|
+
import { readFile as readFile5 } from "fs/promises";
|
|
3019
|
+
async function loadNormalizedChatJsonl(path) {
|
|
3020
|
+
const text = await readFile5(path, "utf8");
|
|
3021
|
+
const rows = [];
|
|
3022
|
+
const jsonLines = [];
|
|
3023
|
+
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
3024
|
+
if (!line.trim()) continue;
|
|
3025
|
+
let value;
|
|
3026
|
+
try {
|
|
3027
|
+
value = JSON.parse(line);
|
|
3028
|
+
} catch (error) {
|
|
3029
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: malformed JSON`, { cause: error });
|
|
3030
|
+
}
|
|
3031
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
3032
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: expected an object`);
|
|
3033
|
+
}
|
|
3034
|
+
const messages = value.messages;
|
|
3035
|
+
if (!Array.isArray(messages) || ![2, 3].includes(messages.length) || messages.some(
|
|
3036
|
+
(message) => !message || typeof message !== "object" || Array.isArray(message) || !["system", "user", "assistant"].includes(String(message.role)) || typeof message.content !== "string"
|
|
3037
|
+
)) {
|
|
3038
|
+
throw new Error(
|
|
3039
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one user message, and one assistant answer`
|
|
3040
|
+
);
|
|
3041
|
+
}
|
|
3042
|
+
const row = { messages };
|
|
3043
|
+
const offset = row.messages.length === 3 ? 1 : 0;
|
|
3044
|
+
if (offset === 1 && row.messages[0]?.role !== "system" || row.messages[offset]?.role !== "user" || !row.messages[offset]?.content.trim() || row.messages[offset + 1]?.role !== "assistant" || !row.messages[offset + 1]?.content.trim()) {
|
|
3045
|
+
throw new Error(
|
|
3046
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one non-empty user message, and one non-empty assistant answer`
|
|
3047
|
+
);
|
|
3048
|
+
}
|
|
3049
|
+
rows.push(row);
|
|
3050
|
+
jsonLines.push(JSON.stringify(row));
|
|
3051
|
+
}
|
|
3052
|
+
if (rows.length === 0) throw new Error(`Chat JSONL contains no examples: ${path}`);
|
|
3053
|
+
return {
|
|
3054
|
+
rows,
|
|
3055
|
+
jsonl: jsonLines.join("\n")
|
|
2425
3056
|
};
|
|
2426
|
-
delete result.TRANSFORMERS_CACHE;
|
|
2427
|
-
delete result.PYTORCH_TRANSFORMERS_CACHE;
|
|
2428
|
-
delete result.PYTORCH_PRETRAINED_BERT_CACHE;
|
|
2429
|
-
return result;
|
|
2430
3057
|
}
|
|
2431
|
-
function
|
|
3058
|
+
async function normalizeChatJsonlForRelocation(path) {
|
|
3059
|
+
return (await loadNormalizedChatJsonl(path)).jsonl;
|
|
3060
|
+
}
|
|
3061
|
+
function buildSystemMessage(spec) {
|
|
3062
|
+
const parts = [];
|
|
3063
|
+
if (spec.system_prompt.trim()) parts.push(spec.system_prompt.trim());
|
|
3064
|
+
if (spec.guidelines.length > 0) {
|
|
3065
|
+
parts.push(`Guidelines:
|
|
3066
|
+
${spec.guidelines.map((guideline) => `- ${guideline}`).join("\n")}`);
|
|
3067
|
+
}
|
|
3068
|
+
if (spec.constraints.length > 0) {
|
|
3069
|
+
parts.push(`Constraints:
|
|
3070
|
+
${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
|
|
3071
|
+
}
|
|
3072
|
+
return parts.join("\n\n") || "Follow the demonstrated behavior.";
|
|
3073
|
+
}
|
|
3074
|
+
function exampleToChatRow(spec, example) {
|
|
3075
|
+
const system = buildSystemMessage(spec);
|
|
2432
3076
|
return {
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
3077
|
+
messages: [
|
|
3078
|
+
{ role: "system", content: system },
|
|
3079
|
+
{ role: "user", content: example.input },
|
|
3080
|
+
{ role: "assistant", content: example.output }
|
|
3081
|
+
]
|
|
3082
|
+
};
|
|
3083
|
+
}
|
|
3084
|
+
function compileSpecToJsonl(spec) {
|
|
3085
|
+
return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
|
|
3086
|
+
}
|
|
3087
|
+
function examplesFromSpec(spec) {
|
|
3088
|
+
return spec.examples;
|
|
3089
|
+
}
|
|
3090
|
+
async function examplesFromChatJsonl(path) {
|
|
3091
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
3092
|
+
const examples = [];
|
|
3093
|
+
for (const [index, row] of normalized.rows.entries()) {
|
|
3094
|
+
const user = row.messages.at(-2);
|
|
3095
|
+
const assistant = row.messages.at(-1);
|
|
3096
|
+
examples.push({
|
|
3097
|
+
input: user.content,
|
|
3098
|
+
output: assistant.content
|
|
3099
|
+
});
|
|
3100
|
+
}
|
|
3101
|
+
return examples;
|
|
3102
|
+
}
|
|
3103
|
+
async function evaluationSuiteFromChatJsonl(path, configuredSystem) {
|
|
3104
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
3105
|
+
const inputIdentities = /* @__PURE__ */ new Set();
|
|
3106
|
+
for (const row of normalized.rows) {
|
|
3107
|
+
const input = row.messages.at(-2).content;
|
|
3108
|
+
const identity = input.trim().replace(/\s+/g, " ").toLowerCase();
|
|
3109
|
+
if (inputIdentities.has(identity)) {
|
|
3110
|
+
throw new Error(
|
|
3111
|
+
"General regression data contains duplicate inputs; evaluation prompts must be unique."
|
|
3112
|
+
);
|
|
3113
|
+
}
|
|
3114
|
+
inputIdentities.add(identity);
|
|
3115
|
+
}
|
|
3116
|
+
const systems = new Set(
|
|
3117
|
+
normalized.rows.map((row) => row.messages.find((message) => message.role === "system")?.content).filter((value) => value !== void 0)
|
|
3118
|
+
);
|
|
3119
|
+
if (systems.size > 1) {
|
|
3120
|
+
throw new Error("General regression rows must use one consistent system message.");
|
|
3121
|
+
}
|
|
3122
|
+
const embeddedSystem = [...systems][0];
|
|
3123
|
+
if (configuredSystem !== void 0 && embeddedSystem !== void 0 && configuredSystem !== embeddedSystem) {
|
|
3124
|
+
throw new Error(
|
|
3125
|
+
"evaluation.generalRegression.systemPrompt must match the system message embedded in the dataset."
|
|
3126
|
+
);
|
|
3127
|
+
}
|
|
3128
|
+
return {
|
|
3129
|
+
examples: normalized.rows.map((row) => ({
|
|
3130
|
+
input: row.messages.at(-2).content,
|
|
3131
|
+
output: row.messages.at(-1).content
|
|
3132
|
+
})),
|
|
3133
|
+
system: configuredSystem ?? embeddedSystem ?? "You are a helpful assistant."
|
|
2436
3134
|
};
|
|
2437
3135
|
}
|
|
2438
3136
|
|
|
2439
3137
|
// src/local-runtime/prefetch.ts
|
|
3138
|
+
import { createHash as createHash4 } from "crypto";
|
|
3139
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
3140
|
+
import { lstat as lstat4, mkdir as mkdir6, readFile as readFile6, readdir as readdir4, realpath as realpath2, stat as stat4 } from "fs/promises";
|
|
3141
|
+
import { isAbsolute as isAbsolute4, join as join8, relative as relative2, resolve as resolve10, sep as sep3 } from "path";
|
|
2440
3142
|
var NEMOTRON_BF16_FILE_DIGESTS = {
|
|
2441
3143
|
".gitattributes": { algorithm: "sha1", digest: "fe3f694ce14b577c9833fca6e83c5e26a7119127" },
|
|
2442
3144
|
"LICENSE": { algorithm: "sha1", digest: "4d76cc87c9edb280753e3bee3b499ae461113835" },
|
|
@@ -2508,8 +3210,8 @@ function isStrictPhysicalDescendant(parent, child) {
|
|
|
2508
3210
|
async function verifySnapshotCacheLocation(path, modelId, revision, modelCache) {
|
|
2509
3211
|
const repository = `models--${modelId.replaceAll("/", "--")}`;
|
|
2510
3212
|
const layout = resolveHuggingFaceCacheLayout(modelCache);
|
|
2511
|
-
const repositoryRoot =
|
|
2512
|
-
const expectedPath =
|
|
3213
|
+
const repositoryRoot = join8(layout.hubCache, repository);
|
|
3214
|
+
const expectedPath = join8(repositoryRoot, "snapshots", revision);
|
|
2513
3215
|
const [
|
|
2514
3216
|
physicalCacheRoot,
|
|
2515
3217
|
physicalHubRoot,
|
|
@@ -2521,7 +3223,7 @@ async function verifySnapshotCacheLocation(path, modelId, revision, modelCache)
|
|
|
2521
3223
|
realpath2(layout.hfHome).catch(() => null),
|
|
2522
3224
|
realpath2(layout.hubCache).catch(() => null),
|
|
2523
3225
|
realpath2(repositoryRoot).catch(() => null),
|
|
2524
|
-
realpath2(
|
|
3226
|
+
realpath2(join8(repositoryRoot, "blobs")).catch(() => null),
|
|
2525
3227
|
realpath2(path).catch(() => null),
|
|
2526
3228
|
realpath2(expectedPath).catch(() => null)
|
|
2527
3229
|
]);
|
|
@@ -2537,7 +3239,7 @@ async function verifySnapshotCacheLocation(path, modelId, revision, modelCache)
|
|
|
2537
3239
|
throw new Error(`Hugging Face repository escapes its configured hub cache: ${repositoryRoot}`);
|
|
2538
3240
|
}
|
|
2539
3241
|
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalBlobRoot)) {
|
|
2540
|
-
throw new Error(`Hugging Face blob store escapes its repository: ${
|
|
3242
|
+
throw new Error(`Hugging Face blob store escapes its repository: ${join8(repositoryRoot, "blobs")}`);
|
|
2541
3243
|
}
|
|
2542
3244
|
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalExpectedPath)) {
|
|
2543
3245
|
throw new Error(`Hugging Face snapshot escapes its configured repository: ${expectedPath}`);
|
|
@@ -2599,13 +3301,13 @@ async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelC
|
|
|
2599
3301
|
const expectedFiles = new Set(Object.keys(digests));
|
|
2600
3302
|
const unexpected = [...actualFiles].filter((name) => !expectedFiles.has(name)).sort();
|
|
2601
3303
|
if (unexpected.length > 0) {
|
|
2602
|
-
throw new Error(`Certified snapshot contains unexpected file: ${
|
|
3304
|
+
throw new Error(`Certified snapshot contains unexpected file: ${join8(path, unexpected[0])}`);
|
|
2603
3305
|
}
|
|
2604
3306
|
for (const [name, expected] of Object.entries(digests)) {
|
|
2605
3307
|
if (!actualFiles.has(name)) {
|
|
2606
|
-
throw new Error(`Certified snapshot file is missing: ${
|
|
3308
|
+
throw new Error(`Certified snapshot file is missing: ${join8(path, name)}`);
|
|
2607
3309
|
}
|
|
2608
|
-
const file =
|
|
3310
|
+
const file = join8(path, name);
|
|
2609
3311
|
const physicalFile = physicalFiles.get(file);
|
|
2610
3312
|
if (await certifiedFileDigest(physicalFile, expected) !== expected.digest) {
|
|
2611
3313
|
throw new Error(`Certified snapshot file checksum mismatch: ${file}`);
|
|
@@ -2624,7 +3326,7 @@ function buildModelPrefetchPayload(request, config) {
|
|
|
2624
3326
|
return {
|
|
2625
3327
|
base_model: model.id,
|
|
2626
3328
|
...revision ? { revision } : {},
|
|
2627
|
-
...config.paths.modelCache ? { model_cache:
|
|
3329
|
+
...config.paths.modelCache ? { model_cache: resolve10(config.paths.modelCache) } : {}
|
|
2628
3330
|
};
|
|
2629
3331
|
}
|
|
2630
3332
|
function safeName(value) {
|
|
@@ -2648,7 +3350,7 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2648
3350
|
const files = [];
|
|
2649
3351
|
const visit = async (directory) => {
|
|
2650
3352
|
for (const entry of await readdir4(directory, { withFileTypes: true })) {
|
|
2651
|
-
const child =
|
|
3353
|
+
const child = join8(directory, entry.name);
|
|
2652
3354
|
if (entry.isSymbolicLink()) {
|
|
2653
3355
|
const metadata2 = await stat4(child);
|
|
2654
3356
|
if (!metadata2.isFile()) {
|
|
@@ -2672,15 +3374,15 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2672
3374
|
throw new Error(`Local base-model directory contains no non-empty Transformers model weights: ${path}`);
|
|
2673
3375
|
}
|
|
2674
3376
|
const required = async (name) => {
|
|
2675
|
-
const metadata = await stat4(
|
|
3377
|
+
const metadata = await stat4(join8(path, name)).catch(() => null);
|
|
2676
3378
|
return Boolean(metadata?.isFile() && metadata.size > 0);
|
|
2677
3379
|
};
|
|
2678
3380
|
if (!await required("config.json")) throw new Error(`Local base-model directory is missing config.json: ${path}`);
|
|
2679
3381
|
try {
|
|
2680
|
-
const config = JSON.parse(await
|
|
3382
|
+
const config = JSON.parse(await readFile6(join8(path, "config.json"), "utf8"));
|
|
2681
3383
|
assertCertifiedBaseModelConfig(
|
|
2682
3384
|
config,
|
|
2683
|
-
`Local base-model config ${
|
|
3385
|
+
`Local base-model config ${join8(path, "config.json")}`,
|
|
2684
3386
|
expectedModelId
|
|
2685
3387
|
);
|
|
2686
3388
|
} catch (error) {
|
|
@@ -2710,7 +3412,7 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2710
3412
|
throw new Error(`Local base-model directory is missing tokenizer metadata or vocabulary: ${path}`);
|
|
2711
3413
|
}
|
|
2712
3414
|
for (const file of files.filter((candidate) => candidate.path.endsWith(".index.json"))) {
|
|
2713
|
-
const parsed = JSON.parse(await
|
|
3415
|
+
const parsed = JSON.parse(await readFile6(file.path, "utf8"));
|
|
2714
3416
|
if (!parsed.weight_map || typeof parsed.weight_map !== "object" || Array.isArray(parsed.weight_map)) {
|
|
2715
3417
|
throw new Error(`Local base-model weight index is invalid: ${file.path}`);
|
|
2716
3418
|
}
|
|
@@ -2730,14 +3432,14 @@ async function prefetchBaseModel(args) {
|
|
|
2730
3432
|
...buildModelPrefetchPayload(args.request, args.config),
|
|
2731
3433
|
...args.localOnly ? { local_files_only: true } : {}
|
|
2732
3434
|
};
|
|
2733
|
-
const artifactDir =
|
|
3435
|
+
const artifactDir = resolve10(
|
|
2734
3436
|
args.config.artifactRoot,
|
|
2735
3437
|
args.localOnly ? "verify-base" : "prefetch",
|
|
2736
3438
|
`${safeName(args.request.spec_snapshot.base_model)}-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
|
|
2737
3439
|
);
|
|
2738
|
-
await
|
|
3440
|
+
await mkdir6(artifactDir, { recursive: true });
|
|
2739
3441
|
if (args.config.paths.baseModel) {
|
|
2740
|
-
const localPath =
|
|
3442
|
+
const localPath = resolve10(args.config.paths.baseModel);
|
|
2741
3443
|
const verified = await verifyLocalBaseModel(
|
|
2742
3444
|
localPath,
|
|
2743
3445
|
args.request.spec_snapshot.base_model,
|
|
@@ -2754,9 +3456,9 @@ async function prefetchBaseModel(args) {
|
|
|
2754
3456
|
reason: args.localOnly ? "Verified the configured local base-model artifact; no network access was used." : "paths.baseModel points at a verified local base-model artifact; no Hugging Face download is needed."
|
|
2755
3457
|
};
|
|
2756
3458
|
}
|
|
2757
|
-
const inputPath =
|
|
2758
|
-
const outputPath =
|
|
2759
|
-
const logPath =
|
|
3459
|
+
const inputPath = join8(artifactDir, "prefetch-input.json");
|
|
3460
|
+
const outputPath = join8(artifactDir, "prefetch-output.json");
|
|
3461
|
+
const logPath = join8(artifactDir, "prefetch.log");
|
|
2760
3462
|
await writeJson(inputPath, payload);
|
|
2761
3463
|
const entrypoint = buildBundledPythonCommand(
|
|
2762
3464
|
"prefetch.py",
|
|
@@ -2793,7 +3495,7 @@ async function prefetchBaseModel(args) {
|
|
|
2793
3495
|
if (exitCode !== 0) {
|
|
2794
3496
|
throw new Error(`Model prefetch exited with code ${exitCode}. See ${logPath}.`);
|
|
2795
3497
|
}
|
|
2796
|
-
const output = JSON.parse(await
|
|
3498
|
+
const output = JSON.parse(await readFile6(outputPath, "utf8"));
|
|
2797
3499
|
const snapshotRevision = baseModelRevisionSchema.safeParse(output.snapshot_revision);
|
|
2798
3500
|
if (!snapshotRevision.success) {
|
|
2799
3501
|
throw new Error(
|
|
@@ -2876,17 +3578,17 @@ function evaluateGeneralRegressionGate(comparison, policy) {
|
|
|
2876
3578
|
}
|
|
2877
3579
|
|
|
2878
3580
|
// src/local-runtime/orchestrator.ts
|
|
2879
|
-
import { lstat as lstat5, mkdir as
|
|
3581
|
+
import { lstat as lstat5, mkdir as mkdir9, open as open2, readFile as readFile9, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
|
|
2880
3582
|
import { createReadStream as createReadStream3 } from "fs";
|
|
2881
3583
|
import { createHash as createHash6, randomUUID as randomUUID4 } from "crypto";
|
|
2882
3584
|
import { performance } from "perf_hooks";
|
|
2883
|
-
import { homedir as
|
|
2884
|
-
import { basename, dirname as
|
|
3585
|
+
import { homedir as homedir5 } from "os";
|
|
3586
|
+
import { basename, dirname as dirname10, join as join11, relative as relative3, resolve as resolve13 } from "path";
|
|
2885
3587
|
|
|
2886
3588
|
// src/local-runtime/evaluation.ts
|
|
2887
3589
|
import { createHash as createHash5 } from "crypto";
|
|
2888
|
-
import { mkdir as
|
|
2889
|
-
import { dirname as
|
|
3590
|
+
import { mkdir as mkdir7, readFile as readFile7, rm as rm2, writeFile as writeFile5 } from "fs/promises";
|
|
3591
|
+
import { dirname as dirname9, join as join9, resolve as resolve11 } from "path";
|
|
2890
3592
|
function normalize(value) {
|
|
2891
3593
|
return value.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2892
3594
|
}
|
|
@@ -3106,8 +3808,8 @@ async function runTransformersInference(args) {
|
|
|
3106
3808
|
const inputPath = `${args.outputPath}.inference-input.json`;
|
|
3107
3809
|
const outputPath = `${args.outputPath}.inference-output.json`;
|
|
3108
3810
|
const logPath = `${args.outputPath}.inference.log`;
|
|
3109
|
-
await
|
|
3110
|
-
await
|
|
3811
|
+
await mkdir7(dirname9(inputPath), { recursive: true });
|
|
3812
|
+
await writeFile5(inputPath, `${JSON.stringify({
|
|
3111
3813
|
protocol_version: INFERENCE_PROTOCOL_VERSION,
|
|
3112
3814
|
kind: args.kind,
|
|
3113
3815
|
model_id: args.modelId,
|
|
@@ -3121,7 +3823,7 @@ async function runTransformersInference(args) {
|
|
|
3121
3823
|
id: String(index),
|
|
3122
3824
|
input: example.input
|
|
3123
3825
|
})),
|
|
3124
|
-
model_cache: args.config.paths.modelCache ?
|
|
3826
|
+
model_cache: args.config.paths.modelCache ? resolve11(args.config.paths.modelCache) : void 0,
|
|
3125
3827
|
trust_remote_code: false,
|
|
3126
3828
|
device: args.config.evaluation.inference.device,
|
|
3127
3829
|
generation: {
|
|
@@ -3170,7 +3872,7 @@ async function runTransformersInference(args) {
|
|
|
3170
3872
|
}
|
|
3171
3873
|
let output;
|
|
3172
3874
|
try {
|
|
3173
|
-
output = JSON.parse(await
|
|
3875
|
+
output = JSON.parse(await readFile7(outputPath, "utf8"));
|
|
3174
3876
|
} catch (error) {
|
|
3175
3877
|
throw new Error(
|
|
3176
3878
|
`Transformers inference did not write valid JSON: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -3207,15 +3909,15 @@ function baselineInputsAreStable(args) {
|
|
|
3207
3909
|
return args.config.paths.baseModel ? Boolean(args.sourceFingerprint) : Boolean(args.baseModelRevision);
|
|
3208
3910
|
}
|
|
3209
3911
|
function baselineStoreRoot(config) {
|
|
3210
|
-
return config.storeRoot ?
|
|
3912
|
+
return config.storeRoot ? resolve11(config.storeRoot) : defaultLocalHome();
|
|
3211
3913
|
}
|
|
3212
3914
|
function baselineCachePath(config, key) {
|
|
3213
|
-
return
|
|
3915
|
+
return join9(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
|
|
3214
3916
|
}
|
|
3215
3917
|
async function readBaselineCache(config, path) {
|
|
3216
3918
|
await hardenExistingLocalStore(baselineStoreRoot(config));
|
|
3217
3919
|
try {
|
|
3218
|
-
return evalReportSchema.parse(JSON.parse(await
|
|
3920
|
+
return evalReportSchema.parse(JSON.parse(await readFile7(path, "utf8")));
|
|
3219
3921
|
} catch {
|
|
3220
3922
|
return null;
|
|
3221
3923
|
}
|
|
@@ -3224,7 +3926,7 @@ var cachedPackageVersion = null;
|
|
|
3224
3926
|
async function packageVersion() {
|
|
3225
3927
|
if (cachedPackageVersion !== null) return cachedPackageVersion;
|
|
3226
3928
|
try {
|
|
3227
|
-
const raw = await
|
|
3929
|
+
const raw = await readFile7(resolve11(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
|
|
3228
3930
|
cachedPackageVersion = String(
|
|
3229
3931
|
JSON.parse(raw).version ?? "unknown"
|
|
3230
3932
|
);
|
|
@@ -3293,7 +3995,7 @@ async function evaluateExamples(args) {
|
|
|
3293
3995
|
baseModelId,
|
|
3294
3996
|
args.baseModelRevision
|
|
3295
3997
|
);
|
|
3296
|
-
const localModelSource = args.modelSource ?
|
|
3998
|
+
const localModelSource = args.modelSource ? resolve11(args.modelSource) !== resolve11(baseModelId) : false;
|
|
3297
3999
|
const baseModelRevision = localModelSource ? defaultBaseModelRevision(baseModelId) : requestedBaseModelRevision;
|
|
3298
4000
|
const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
|
|
3299
4001
|
const truncated = args.examples.length > maxExamples;
|
|
@@ -3409,8 +4111,8 @@ function compareEvalReports(baseline, candidate) {
|
|
|
3409
4111
|
}
|
|
3410
4112
|
|
|
3411
4113
|
// src/local-runtime/process-training.ts
|
|
3412
|
-
import { access, mkdir as
|
|
3413
|
-
import { join as
|
|
4114
|
+
import { access as access2, mkdir as mkdir8, readFile as readFile8, writeFile as writeFile6 } from "fs/promises";
|
|
4115
|
+
import { join as join10, resolve as resolve12 } from "path";
|
|
3414
4116
|
function buildTrainingHyperparameters(request, options = {}) {
|
|
3415
4117
|
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
3416
4118
|
const hyper = request.hyperparameters;
|
|
@@ -3434,7 +4136,7 @@ function buildTrainingHyperparameters(request, options = {}) {
|
|
|
3434
4136
|
}
|
|
3435
4137
|
async function pathExists(path) {
|
|
3436
4138
|
try {
|
|
3437
|
-
await
|
|
4139
|
+
await access2(path);
|
|
3438
4140
|
return true;
|
|
3439
4141
|
} catch {
|
|
3440
4142
|
return false;
|
|
@@ -3501,15 +4203,15 @@ function createTrainingProgressForwarder(reporter) {
|
|
|
3501
4203
|
async function launchProcessTraining(args) {
|
|
3502
4204
|
const { request, artifacts, config } = args;
|
|
3503
4205
|
const jobName = `tt-local-${request.run_id}`;
|
|
3504
|
-
await
|
|
4206
|
+
await mkdir8(artifacts.trainingDir, { recursive: true });
|
|
3505
4207
|
await copyDatasetToTrainingChannel(artifacts);
|
|
3506
4208
|
const hyperparameters = buildTrainingHyperparameters(request, {
|
|
3507
4209
|
baseModelRevision: args.baseModelRevision
|
|
3508
4210
|
});
|
|
3509
|
-
await writeJson(
|
|
4211
|
+
await writeJson(join10(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
|
|
3510
4212
|
const entrypoint = buildBundledPythonCommand("train.py");
|
|
3511
4213
|
if (config.dryRun) {
|
|
3512
|
-
await
|
|
4214
|
+
await writeFile6(
|
|
3513
4215
|
artifacts.trainingLog,
|
|
3514
4216
|
"Dry run enabled. Training process was not launched.\n",
|
|
3515
4217
|
"utf8"
|
|
@@ -3531,20 +4233,20 @@ async function launchProcessTraining(args) {
|
|
|
3531
4233
|
withOfflineHuggingFaceCacheEnvironment({
|
|
3532
4234
|
...inheritedEnv,
|
|
3533
4235
|
BACKEND: "local",
|
|
3534
|
-
SM_CHANNEL_TRAINING:
|
|
3535
|
-
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ?
|
|
3536
|
-
SM_MODEL_DIR:
|
|
3537
|
-
SM_OUTPUT_DIR:
|
|
3538
|
-
TT_HYPERPARAMETERS_PATH:
|
|
4236
|
+
SM_CHANNEL_TRAINING: resolve12(artifacts.trainingInputDir),
|
|
4237
|
+
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve12(config.paths.baseModel) : void 0,
|
|
4238
|
+
SM_MODEL_DIR: resolve12(artifacts.trainingModelDir),
|
|
4239
|
+
SM_OUTPUT_DIR: resolve12(artifacts.trainingOutputDir),
|
|
4240
|
+
TT_HYPERPARAMETERS_PATH: resolve12(artifacts.trainingConfigDir, "hyperparameters.json")
|
|
3539
4241
|
}, config.paths.modelCache)
|
|
3540
4242
|
);
|
|
3541
4243
|
for (const [key, value] of Object.entries(childEnv)) {
|
|
3542
4244
|
if (value === void 0) delete childEnv[key];
|
|
3543
4245
|
}
|
|
3544
|
-
if (config.paths.modelCache) await
|
|
4246
|
+
if (config.paths.modelCache) await mkdir8(resolve12(config.paths.modelCache), { recursive: true });
|
|
3545
4247
|
if (config.paths.baseModel) {
|
|
3546
4248
|
await verifyLocalBaseModel(
|
|
3547
|
-
|
|
4249
|
+
resolve12(config.paths.baseModel),
|
|
3548
4250
|
request.spec_snapshot.base_model,
|
|
3549
4251
|
config.paths.modelCache
|
|
3550
4252
|
);
|
|
@@ -3571,12 +4273,12 @@ async function launchProcessTraining(args) {
|
|
|
3571
4273
|
onLine: (line) => forwardTrainingProgress(line),
|
|
3572
4274
|
shouldCancel: args.shouldCancel
|
|
3573
4275
|
});
|
|
3574
|
-
const metricsPath =
|
|
3575
|
-
const metrics = await pathExists(metricsPath) ? JSON.parse(await
|
|
4276
|
+
const metricsPath = join10(artifacts.trainingModelDir, "training-metrics.json");
|
|
4277
|
+
const metrics = await pathExists(metricsPath) ? JSON.parse(await readFile8(metricsPath, "utf8")) : null;
|
|
3576
4278
|
if (exitCode !== 0) {
|
|
3577
4279
|
throw new Error(`Training process exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
|
|
3578
4280
|
}
|
|
3579
|
-
const modelTarPath =
|
|
4281
|
+
const modelTarPath = join10(artifacts.trainingOutputDir, "model.tar.gz");
|
|
3580
4282
|
const modelUri = await pathExists(modelTarPath) ? fileUri(modelTarPath) : fileUri(artifacts.trainingModelDir);
|
|
3581
4283
|
await args.reporter?.onEvent?.({
|
|
3582
4284
|
stage: "training",
|
|
@@ -3671,21 +4373,21 @@ function validateLocalPipeline(pipeline) {
|
|
|
3671
4373
|
return pipeline;
|
|
3672
4374
|
}
|
|
3673
4375
|
async function loadJsonFile(path) {
|
|
3674
|
-
return JSON.parse(await
|
|
4376
|
+
return JSON.parse(await readFile9(path, "utf8"));
|
|
3675
4377
|
}
|
|
3676
4378
|
async function loadRunRequest(path) {
|
|
3677
4379
|
return fineTuneRunRequestSchema.parse(await loadJsonFile(path));
|
|
3678
4380
|
}
|
|
3679
4381
|
async function loadLocalRunnerConfig(path) {
|
|
3680
4382
|
if (!path) return localRunnerConfigSchema.parse({});
|
|
3681
|
-
const configPath =
|
|
3682
|
-
const base =
|
|
4383
|
+
const configPath = resolve13(path);
|
|
4384
|
+
const base = dirname10(configPath);
|
|
3683
4385
|
const config = localRunnerConfigSchema.parse(await loadJsonFile(configPath));
|
|
3684
4386
|
const configPathValue = (value) => {
|
|
3685
4387
|
if (!value) return void 0;
|
|
3686
|
-
if (value === "~") return
|
|
3687
|
-
if (value.startsWith("~/")) return
|
|
3688
|
-
return
|
|
4388
|
+
if (value === "~") return homedir5();
|
|
4389
|
+
if (value.startsWith("~/")) return resolve13(homedir5(), value.slice(2));
|
|
4390
|
+
return resolve13(base, value);
|
|
3689
4391
|
};
|
|
3690
4392
|
return {
|
|
3691
4393
|
...config,
|
|
@@ -3883,21 +4585,21 @@ async function hashFileIfPresent(path) {
|
|
|
3883
4585
|
}
|
|
3884
4586
|
async function packageVersion2() {
|
|
3885
4587
|
try {
|
|
3886
|
-
const parsed = JSON.parse(await
|
|
4588
|
+
const parsed = JSON.parse(await readFile9(resolve13(packageRoot2, "package.json"), "utf8"));
|
|
3887
4589
|
return typeof parsed.version === "string" ? parsed.version : "unknown";
|
|
3888
4590
|
} catch {
|
|
3889
4591
|
return "unknown";
|
|
3890
4592
|
}
|
|
3891
4593
|
}
|
|
3892
4594
|
async function runtimeFingerprint() {
|
|
3893
|
-
const bundledProject2 =
|
|
4595
|
+
const bundledProject2 = resolve13(packageRoot2, "training/adapter");
|
|
3894
4596
|
return hashJson({
|
|
3895
4597
|
tt_local_version: await packageVersion2(),
|
|
3896
4598
|
node_version: process.version,
|
|
3897
4599
|
platform: process.platform,
|
|
3898
4600
|
architecture: process.arch,
|
|
3899
|
-
runner_pyproject: await hashFileIfPresent(
|
|
3900
|
-
runner_uv_lock: await hashFileIfPresent(
|
|
4601
|
+
runner_pyproject: await hashFileIfPresent(resolve13(bundledProject2, "pyproject.toml")),
|
|
4602
|
+
runner_uv_lock: await hashFileIfPresent(resolve13(bundledProject2, "uv.lock"))
|
|
3901
4603
|
});
|
|
3902
4604
|
}
|
|
3903
4605
|
function preparedSourceFingerprint(args) {
|
|
@@ -3944,9 +4646,9 @@ async function resolveBaseModelRevision(request, config) {
|
|
|
3944
4646
|
}
|
|
3945
4647
|
const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
|
|
3946
4648
|
const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
|
|
3947
|
-
const refPath =
|
|
4649
|
+
const refPath = resolve13(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
|
|
3948
4650
|
try {
|
|
3949
|
-
const revision = (await
|
|
4651
|
+
const revision = (await readFile9(refPath, "utf8")).trim();
|
|
3950
4652
|
if (revision) return revision;
|
|
3951
4653
|
} catch {
|
|
3952
4654
|
}
|
|
@@ -3965,7 +4667,7 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
|
|
|
3965
4667
|
const files = [];
|
|
3966
4668
|
const visit = async (directory) => {
|
|
3967
4669
|
for (const entry of await readdir5(directory, { withFileTypes: true })) {
|
|
3968
|
-
const child =
|
|
4670
|
+
const child = resolve13(directory, entry.name);
|
|
3969
4671
|
if (entry.isDirectory()) {
|
|
3970
4672
|
await visit(child);
|
|
3971
4673
|
continue;
|
|
@@ -4004,12 +4706,12 @@ async function readStageMetadata(path) {
|
|
|
4004
4706
|
}
|
|
4005
4707
|
}
|
|
4006
4708
|
async function removePrefixedArtifacts(path) {
|
|
4007
|
-
const directory =
|
|
4709
|
+
const directory = dirname10(path);
|
|
4008
4710
|
const prefix = `${basename(path)}.`;
|
|
4009
4711
|
const names = await readdir5(directory).catch(() => []);
|
|
4010
4712
|
await Promise.all([
|
|
4011
4713
|
rm3(path, { force: true }),
|
|
4012
|
-
...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(
|
|
4714
|
+
...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(resolve13(directory, name), { recursive: true, force: true }))
|
|
4013
4715
|
]);
|
|
4014
4716
|
}
|
|
4015
4717
|
async function cleanupStageArtifacts(artifacts, stage) {
|
|
@@ -4022,7 +4724,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
4022
4724
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
4023
4725
|
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
4024
4726
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
4025
|
-
rm3(
|
|
4727
|
+
rm3(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
4026
4728
|
removeReport(),
|
|
4027
4729
|
rm3(artifacts.artifactManifestJson, { force: true })
|
|
4028
4730
|
]);
|
|
@@ -4040,7 +4742,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
4040
4742
|
await Promise.all([
|
|
4041
4743
|
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
4042
4744
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
4043
|
-
rm3(
|
|
4745
|
+
rm3(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
4044
4746
|
removePrefixedArtifacts(artifacts.candidateEvalJson),
|
|
4045
4747
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
4046
4748
|
removeReport()
|
|
@@ -4076,7 +4778,7 @@ async function throwIfCancelled(store, request) {
|
|
|
4076
4778
|
}
|
|
4077
4779
|
}
|
|
4078
4780
|
async function acquireWorkflowLock(lockPath, description) {
|
|
4079
|
-
await
|
|
4781
|
+
await mkdir9(dirname10(lockPath), { recursive: true });
|
|
4080
4782
|
const token = randomUUID4();
|
|
4081
4783
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4082
4784
|
try {
|
|
@@ -4089,7 +4791,7 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
4089
4791
|
}
|
|
4090
4792
|
return async () => {
|
|
4091
4793
|
try {
|
|
4092
|
-
const owner = JSON.parse(await
|
|
4794
|
+
const owner = JSON.parse(await readFile9(lockPath, "utf8"));
|
|
4093
4795
|
if (owner.token === token) await rm3(lockPath, { force: true });
|
|
4094
4796
|
} catch {
|
|
4095
4797
|
}
|
|
@@ -4099,7 +4801,7 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
4099
4801
|
const metadata = await stat5(lockPath).catch(() => null);
|
|
4100
4802
|
let owner = null;
|
|
4101
4803
|
try {
|
|
4102
|
-
owner = JSON.parse(await
|
|
4804
|
+
owner = JSON.parse(await readFile9(lockPath, "utf8"));
|
|
4103
4805
|
} catch {
|
|
4104
4806
|
owner = null;
|
|
4105
4807
|
}
|
|
@@ -4131,11 +4833,11 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
4131
4833
|
}
|
|
4132
4834
|
async function acquireRunLock(store, runId) {
|
|
4133
4835
|
await store.ensure();
|
|
4134
|
-
return acquireWorkflowLock(
|
|
4836
|
+
return acquireWorkflowLock(resolve13(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
|
|
4135
4837
|
}
|
|
4136
4838
|
async function acquireArtifactLock(artifacts) {
|
|
4137
4839
|
return acquireWorkflowLock(
|
|
4138
|
-
|
|
4840
|
+
join11(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
|
|
4139
4841
|
`Artifact directory ${artifacts.runDir}`
|
|
4140
4842
|
);
|
|
4141
4843
|
}
|
|
@@ -4182,13 +4884,13 @@ function stageFingerprintPath(prepared, stage) {
|
|
|
4182
4884
|
return `${stageOutputPath(prepared, stage)}.stage.json`;
|
|
4183
4885
|
}
|
|
4184
4886
|
async function runnerFileFingerprints() {
|
|
4185
|
-
const project =
|
|
4186
|
-
const bundledSource =
|
|
4187
|
-
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) =>
|
|
4887
|
+
const project = resolve13(packageRoot2, "training/adapter");
|
|
4888
|
+
const bundledSource = resolve13(packageRoot2, "training/adapter/src");
|
|
4889
|
+
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve13(bundledSource, entry.name));
|
|
4188
4890
|
const candidates = [.../* @__PURE__ */ new Set([
|
|
4189
4891
|
...pythonSources,
|
|
4190
|
-
|
|
4191
|
-
|
|
4892
|
+
resolve13(project, "pyproject.toml"),
|
|
4893
|
+
resolve13(project, "uv.lock")
|
|
4192
4894
|
])];
|
|
4193
4895
|
const fingerprints = {};
|
|
4194
4896
|
for (const path of candidates.sort()) fingerprints[path] = await hashFileIfPresent(path);
|
|
@@ -4302,7 +5004,7 @@ async function ensureRunRecord(args) {
|
|
|
4302
5004
|
`Run ${args.request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
|
|
4303
5005
|
);
|
|
4304
5006
|
}
|
|
4305
|
-
if (
|
|
5007
|
+
if (resolve13(existing.artifact_dir) !== resolve13(args.artifacts.runDir)) {
|
|
4306
5008
|
throw new Error(
|
|
4307
5009
|
`Run ${args.request.run_id} is already bound to artifact directory ${existing.artifact_dir}; it cannot be resumed with ${args.artifacts.runDir}.`
|
|
4308
5010
|
);
|
|
@@ -4409,7 +5111,7 @@ async function prepareStage(args) {
|
|
|
4409
5111
|
const preparedExists = await pathExists2(args.artifacts.stageMetadataJson) && await pathExists2(args.artifacts.trainingJsonl);
|
|
4410
5112
|
const prepared = await computePreparedRun({ ...args, writeArtifacts: false });
|
|
4411
5113
|
await args.store.syncRunRequest(args.request, args.artifacts.runDir);
|
|
4412
|
-
await writeJsonAtomic(
|
|
5114
|
+
await writeJsonAtomic(resolve13(args.artifacts.runDir, "request.json"), args.request);
|
|
4413
5115
|
const existingMetadata = preparedExists ? await readStageMetadata(args.artifacts.stageMetadataJson) : null;
|
|
4414
5116
|
let canReuse = preparedExists && existingMetadata?.source_fingerprint === prepared.metadata.source_fingerprint;
|
|
4415
5117
|
if (canReuse) {
|
|
@@ -4938,7 +5640,7 @@ async function runLocalPipeline(input) {
|
|
|
4938
5640
|
if (step.uses === "evaluate") {
|
|
4939
5641
|
const isBase = step.with.model === "base";
|
|
4940
5642
|
const report = isBase ? await runBaselineStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter }) : await runCandidateStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
|
|
4941
|
-
const reportPath =
|
|
5643
|
+
const reportPath = join11(artifacts.runDir, "evaluations", `${step.id}.json`);
|
|
4942
5644
|
await writeJsonAtomic(reportPath, { ...report, artifact_uri: fileUri(reportPath) });
|
|
4943
5645
|
outputs[step.id] = { report: { ...report, artifact_uri: fileUri(reportPath) }, reportPath };
|
|
4944
5646
|
await refreshArtifactManifest(prepared);
|
|
@@ -5027,6 +5729,23 @@ export {
|
|
|
5027
5729
|
getConfigDir,
|
|
5028
5730
|
getAgentConfigDir,
|
|
5029
5731
|
defaultStoreRoot,
|
|
5732
|
+
warningsFromSnapshot,
|
|
5733
|
+
minimalMachineLearningEnvironment,
|
|
5734
|
+
withHuggingFaceCacheEnvironment,
|
|
5735
|
+
withOfflineHuggingFaceCacheEnvironment,
|
|
5736
|
+
reportInBackground,
|
|
5737
|
+
sanitizeLogLine,
|
|
5738
|
+
forwardStreamLines,
|
|
5739
|
+
localRuntimePackageRoot,
|
|
5740
|
+
buildBundledPythonCommand,
|
|
5741
|
+
withBundledPythonEnvironment,
|
|
5742
|
+
buildFoundationPythonCommand,
|
|
5743
|
+
withFoundationPythonEnvironment,
|
|
5744
|
+
runLoggedProcess,
|
|
5745
|
+
readHardwareSnapshot,
|
|
5746
|
+
formatHostStatusLine,
|
|
5747
|
+
formatAgentHostBlock,
|
|
5748
|
+
assessHardware,
|
|
5030
5749
|
isTerminalRunState,
|
|
5031
5750
|
defaultLocalHome,
|
|
5032
5751
|
localStorePaths,
|
|
@@ -5070,18 +5789,6 @@ export {
|
|
|
5070
5789
|
examplesFromSpec,
|
|
5071
5790
|
examplesFromChatJsonl,
|
|
5072
5791
|
evaluationSuiteFromChatJsonl,
|
|
5073
|
-
reportInBackground,
|
|
5074
|
-
sanitizeLogLine,
|
|
5075
|
-
forwardStreamLines,
|
|
5076
|
-
localRuntimePackageRoot,
|
|
5077
|
-
buildBundledPythonCommand,
|
|
5078
|
-
withBundledPythonEnvironment,
|
|
5079
|
-
buildFoundationPythonCommand,
|
|
5080
|
-
withFoundationPythonEnvironment,
|
|
5081
|
-
runLoggedProcess,
|
|
5082
|
-
minimalMachineLearningEnvironment,
|
|
5083
|
-
withHuggingFaceCacheEnvironment,
|
|
5084
|
-
withOfflineHuggingFaceCacheEnvironment,
|
|
5085
5792
|
tokenF1,
|
|
5086
5793
|
verifyModelPrefetchCacheReport,
|
|
5087
5794
|
buildModelPrefetchPayload,
|
|
@@ -5098,4 +5805,4 @@ export {
|
|
|
5098
5805
|
runLocalPipeline,
|
|
5099
5806
|
runLocalFineTune
|
|
5100
5807
|
};
|
|
5101
|
-
//# sourceMappingURL=chunk-
|
|
5808
|
+
//# sourceMappingURL=chunk-AVMRHDFY.js.map
|