@tuned-tensor/cli 0.13.1-beta.3 → 0.13.1-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/dist/{chunk-2P2RL23O.js → chunk-AVMRHDFY.js} +1978 -1122
- package/dist/chunk-AVMRHDFY.js.map +1 -0
- package/dist/index.js +182 -55
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +62 -9
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/architecture.md +4 -4
- package/docs/local-runtime/spark.md +6 -0
- package/examples/local-runtime/dry-runner.json +2 -2
- package/examples/local-runtime/local-runner.json +2 -2
- package/package.json +5 -5
- package/dist/chunk-2P2RL23O.js.map +0 -1
- /package/training/{local-runner → adapter}/pyproject.toml +0 -0
- /package/training/{local-runner → adapter}/src/evaluate.py +0 -0
- /package/training/{local-runner → adapter}/src/model_contract.py +0 -0
- /package/training/{local-runner → adapter}/src/prefetch.py +0 -0
- /package/training/{local-runner → adapter}/src/serve.py +0 -0
- /package/training/{local-runner → adapter}/src/sft_data.py +0 -0
- /package/training/{local-runner → adapter}/src/train.py +0 -0
- /package/training/{local-runner → adapter}/uv.lock +0 -0
|
@@ -477,37 +477,37 @@ async function verifyTarGzipArchive(path) {
|
|
|
477
477
|
if (sizeText && !/^[0-7]+$/.test(sizeText)) throw new Error(`Invalid tar member size in ${path}.`);
|
|
478
478
|
const size = sizeText ? Number.parseInt(sizeText, 8) : 0;
|
|
479
479
|
if (!Number.isSafeInteger(size) || size < 0) throw new Error(`Invalid tar member size in ${path}.`);
|
|
480
|
-
const
|
|
481
|
-
if (
|
|
482
|
-
throw new Error(`Unsafe or unsupported tar member type ${JSON.stringify(
|
|
480
|
+
const type2 = String.fromCharCode(header[156] ?? 0);
|
|
481
|
+
if (type2 !== "\0" && type2 !== "0" && type2 !== "5" && type2 !== "x" && type2 !== "g") {
|
|
482
|
+
throw new Error(`Unsafe or unsupported tar member type ${JSON.stringify(type2)} in ${path}.`);
|
|
483
483
|
}
|
|
484
484
|
const shortName = header.subarray(0, 100).toString("utf8").replace(/\0.*$/, "");
|
|
485
485
|
const prefix = header.subarray(345, 500).toString("utf8").replace(/\0.*$/, "");
|
|
486
486
|
const headerPath = safeMemberPath(prefix ? `${prefix}/${shortName}` : shortName);
|
|
487
|
-
const memberPath =
|
|
488
|
-
if (
|
|
487
|
+
const memberPath = type2 === "x" || type2 === "g" ? headerPath : safeMemberPath(pendingPaxPath ?? headerPath);
|
|
488
|
+
if (type2 !== "x" && type2 !== "g") pendingPaxPath = void 0;
|
|
489
489
|
const segments = memberPath.split("/").filter(Boolean);
|
|
490
490
|
const name = segments.at(-1)?.toLowerCase();
|
|
491
491
|
const metadataOnly = name === "training-metrics.json" || name === "trainer_state.json" || name === "training_args.bin" || name === "config.json" || name === "generation_config.json" || name === "adapter_config.json" || name === "readme.md" || name?.startsWith("tokenizer") || name?.startsWith("special_tokens") || name?.startsWith("added_tokens") || name === "vocab.json" || name === "merges.txt";
|
|
492
492
|
const recognizedPayload = Boolean(name) && name !== "training_args.bin" && (name.endsWith(".safetensors") || name.endsWith(".bin"));
|
|
493
493
|
const adapterWeight = name === "adapter_model.safetensors" || name === "adapter_model.bin";
|
|
494
494
|
const fullModelWeight = Boolean(name) && (name.endsWith(".safetensors") && !name.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(name));
|
|
495
|
-
if (size > 0 && name === "adapter_config.json" && (
|
|
495
|
+
if (size > 0 && name === "adapter_config.json" && (type2 === "\0" || type2 === "0")) {
|
|
496
496
|
adapterConfigEntries += 1;
|
|
497
497
|
}
|
|
498
|
-
if ((
|
|
498
|
+
if ((type2 === "x" || type2 === "g") && size > maxPaxBytes) {
|
|
499
499
|
throw new Error(`PAX metadata exceeds 1 MiB in ${path}.`);
|
|
500
500
|
}
|
|
501
|
-
if (size > 0 && !metadataOnly && (
|
|
502
|
-
if (size > 0 && recognizedPayload && (
|
|
501
|
+
if (size > 0 && !metadataOnly && (type2 === "\0" || type2 === "0")) payloadEntries += 1;
|
|
502
|
+
if (size > 0 && recognizedPayload && (type2 === "\0" || type2 === "0")) {
|
|
503
503
|
recognizedPayloadEntries += 1;
|
|
504
504
|
recognizedPayloadBytes += size;
|
|
505
505
|
}
|
|
506
|
-
if (size > 0 && adapterWeight && (
|
|
506
|
+
if (size > 0 && adapterWeight && (type2 === "\0" || type2 === "0")) {
|
|
507
507
|
adapterWeightEntries += 1;
|
|
508
508
|
adapterWeightBytes += size;
|
|
509
509
|
}
|
|
510
|
-
if (size > 0 && fullModelWeight && (
|
|
510
|
+
if (size > 0 && fullModelWeight && (type2 === "\0" || type2 === "0")) {
|
|
511
511
|
fullModelWeightEntries += 1;
|
|
512
512
|
fullModelWeightBytes += size;
|
|
513
513
|
}
|
|
@@ -515,7 +515,7 @@ async function verifyTarGzipArchive(path) {
|
|
|
515
515
|
if (expandedBytes > maxExpandedBytes) throw new Error(`Tar archive exceeds 20 GiB expanded size: ${path}.`);
|
|
516
516
|
payloadBytes = size;
|
|
517
517
|
paddingBytes = (512 - size % 512) % 512;
|
|
518
|
-
paxType =
|
|
518
|
+
paxType = type2 === "x" || type2 === "g" ? type2 : null;
|
|
519
519
|
paxChunks = [];
|
|
520
520
|
entries += 1;
|
|
521
521
|
if (entries > maxEntries) throw new Error(`Tar archive exceeds ${maxEntries} members: ${path}.`);
|
|
@@ -834,717 +834,1233 @@ async function assertUsableModelArtifact(uri) {
|
|
|
834
834
|
return inspection;
|
|
835
835
|
}
|
|
836
836
|
|
|
837
|
-
// src/local-runtime/
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
input: z.string().min(1),
|
|
843
|
-
output: z.string().min(1)
|
|
844
|
-
}).strict();
|
|
845
|
-
var specSnapshotSchema = z.object({
|
|
846
|
-
name: z.string().min(1),
|
|
847
|
-
description: z.string().default(""),
|
|
848
|
-
system_prompt: z.string().default(""),
|
|
849
|
-
guidelines: z.array(z.string()).default([]),
|
|
850
|
-
examples: z.array(behaviorSpecExampleSchema).default([]),
|
|
851
|
-
constraints: z.array(z.string()).default([]),
|
|
852
|
-
base_model: z.string().transform((value) => canonicalizeTrainingModel(value))
|
|
853
|
-
}).strict();
|
|
854
|
-
var foundationHyperparametersSchema = z.object({
|
|
855
|
-
vocab_size: z.number().int().min(64).max(65536).default(256),
|
|
856
|
-
max_chars: z.number().int().min(1e3).max(2e9).default(2e4),
|
|
857
|
-
depth: z.number().int().min(1).max(64).default(2),
|
|
858
|
-
pretrain_steps: z.number().int().min(1).max(1e7).default(2),
|
|
859
|
-
finetune_steps: z.number().int().min(1).max(1e7).default(2),
|
|
860
|
-
rl_steps: z.number().int().min(0).max(1e7).default(0),
|
|
861
|
-
batch_size: z.number().int().min(1).max(1024).default(2),
|
|
862
|
-
sequence_length: z.number().int().min(16).max(8192).default(64),
|
|
863
|
-
nproc_per_node: z.number().int().min(1).max(16).default(1)
|
|
864
|
-
}).strict();
|
|
865
|
-
var localFoundationSpecFileSchema = z.object({
|
|
866
|
-
engine: z.literal("foundation"),
|
|
867
|
-
id: z.string().uuid().optional(),
|
|
868
|
-
name: z.string().min(1),
|
|
869
|
-
description: z.string().default(""),
|
|
870
|
-
system_prompt: z.string().default(""),
|
|
871
|
-
guidelines: z.array(z.string()).default([]),
|
|
872
|
-
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
873
|
-
constraints: z.array(z.string()).default([]),
|
|
874
|
-
foundation: foundationHyperparametersSchema
|
|
875
|
-
}).strict();
|
|
876
|
-
var datasetPrebuiltSchema = z.object({
|
|
877
|
-
training: z.string().min(1),
|
|
878
|
-
validation: z.string().min(1).optional(),
|
|
879
|
-
test: z.string().min(1).optional(),
|
|
880
|
-
format: datasetFormatSchema.default("chat_jsonl")
|
|
881
|
-
}).strict().superRefine((dataset, context) => {
|
|
882
|
-
if (!dataset.validation && !dataset.test) {
|
|
883
|
-
context.addIssue({
|
|
884
|
-
code: z.ZodIssueCode.custom,
|
|
885
|
-
path: ["validation"],
|
|
886
|
-
message: "Provide a distinct validation or test split for held-out evaluation"
|
|
887
|
-
});
|
|
837
|
+
// src/local-runtime/run-reporter.ts
|
|
838
|
+
function reportInBackground(action) {
|
|
839
|
+
try {
|
|
840
|
+
void Promise.resolve(action()).catch(() => void 0);
|
|
841
|
+
} catch {
|
|
888
842
|
}
|
|
889
|
-
}
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
}
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
843
|
+
}
|
|
844
|
+
function sanitizeLogLine(line) {
|
|
845
|
+
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
846
|
+
}
|
|
847
|
+
function forwardStreamLines(stream, onLine) {
|
|
848
|
+
let buffered = "";
|
|
849
|
+
stream.setEncoding("utf8");
|
|
850
|
+
stream.on("data", (chunk) => {
|
|
851
|
+
buffered += chunk;
|
|
852
|
+
const lines = buffered.split(/\r\n|\n|\r/);
|
|
853
|
+
buffered = lines.pop() ?? "";
|
|
854
|
+
for (const line of lines) {
|
|
855
|
+
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
856
|
+
}
|
|
857
|
+
});
|
|
858
|
+
stream.on("end", () => {
|
|
859
|
+
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// src/local-runtime/package-root.ts
|
|
864
|
+
import { existsSync } from "fs";
|
|
865
|
+
import { dirname as dirname2, resolve as resolve3 } from "path";
|
|
866
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
867
|
+
function localRuntimePackageRoot(moduleUrl) {
|
|
868
|
+
const moduleDirectory = dirname2(fileURLToPath2(moduleUrl));
|
|
869
|
+
const candidates = [
|
|
870
|
+
dirname2(moduleDirectory),
|
|
871
|
+
dirname2(dirname2(moduleDirectory))
|
|
872
|
+
];
|
|
873
|
+
const root = candidates.find(
|
|
874
|
+
(candidate) => existsSync(resolve3(candidate, "package.json")) && existsSync(resolve3(candidate, "training/adapter"))
|
|
875
|
+
);
|
|
876
|
+
if (!root) {
|
|
877
|
+
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
918
878
|
}
|
|
919
|
-
|
|
920
|
-
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
921
|
-
engine: z.literal("adapter").optional(),
|
|
922
|
-
id: z.string().uuid().optional(),
|
|
923
|
-
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
924
|
-
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
925
|
-
}).strict();
|
|
926
|
-
var localBehaviorSpecFileSchema = z.union([
|
|
927
|
-
localFoundationSpecFileSchema,
|
|
928
|
-
localAdapterSpecFileSchema
|
|
929
|
-
]);
|
|
930
|
-
var evalExampleResultSchema = z.object({
|
|
931
|
-
prompt: z.string(),
|
|
932
|
-
expected: z.string(),
|
|
933
|
-
actual: z.string(),
|
|
934
|
-
passed: z.boolean(),
|
|
935
|
-
score: z.number().min(0).max(1),
|
|
936
|
-
reasoning: z.string().nullable(),
|
|
937
|
-
latency_ms: z.number().int().nonnegative(),
|
|
938
|
-
scored_by: z.enum(["exact_match", "json_fields", "heuristic"]).optional()
|
|
939
|
-
}).strict();
|
|
940
|
-
var jsonFieldMetricsSchema = z.object({
|
|
941
|
-
fields: z.array(z.string()),
|
|
942
|
-
valid_json_count: z.number().int().nonnegative(),
|
|
943
|
-
valid_json_rate: z.number().min(0).max(1),
|
|
944
|
-
schema_match_count: z.number().int().nonnegative(),
|
|
945
|
-
schema_match_rate: z.number().min(0).max(1),
|
|
946
|
-
all_fields_match_count: z.number().int().nonnegative(),
|
|
947
|
-
all_fields_match_rate: z.number().min(0).max(1),
|
|
948
|
-
field_accuracy: z.record(z.string(), z.object({
|
|
949
|
-
correct: z.number().int().nonnegative(),
|
|
950
|
-
total: z.number().int().nonnegative(),
|
|
951
|
-
accuracy: z.number().min(0).max(1)
|
|
952
|
-
}).strict())
|
|
953
|
-
}).strict();
|
|
954
|
-
var evalSplitSchema = z.enum([
|
|
955
|
-
"spec_examples",
|
|
956
|
-
"spec_holdout",
|
|
957
|
-
"prebuilt_test",
|
|
958
|
-
"prebuilt_validation",
|
|
959
|
-
"general_regression"
|
|
960
|
-
]);
|
|
961
|
-
var evalReportSchema = z.object({
|
|
962
|
-
kind: z.enum(["baseline", "candidate"]),
|
|
963
|
-
model_id: z.string(),
|
|
964
|
-
total: z.number().int().nonnegative(),
|
|
965
|
-
eval_examples_total: z.number().int().nonnegative(),
|
|
966
|
-
eval_examples_used: z.number().int().nonnegative(),
|
|
967
|
-
eval_truncated: z.boolean(),
|
|
968
|
-
eval_split: evalSplitSchema.optional(),
|
|
969
|
-
eval_sample_seed: z.number().int().nullable().optional(),
|
|
970
|
-
avg_score: z.number().min(0).max(1),
|
|
971
|
-
pass_rate: z.number().min(0).max(1),
|
|
972
|
-
exact_match_rate: z.number().min(0).max(1),
|
|
973
|
-
avg_token_f1: z.number().min(0).max(1).optional(),
|
|
974
|
-
avg_latency_ms: z.number().int().nonnegative(),
|
|
975
|
-
cached: z.boolean().optional(),
|
|
976
|
-
cache_key: z.string().optional(),
|
|
977
|
-
results: z.array(evalExampleResultSchema),
|
|
978
|
-
artifact_uri: z.string(),
|
|
979
|
-
scoring_method: z.enum(["heuristic", "exact_match", "json_fields"]),
|
|
980
|
-
inference_provider: z.enum(["none", "transformers"]).optional(),
|
|
981
|
-
scoring_mode: z.enum(["exact_match", "json_fields"]).optional(),
|
|
982
|
-
json_field_metrics: jsonFieldMetricsSchema.optional(),
|
|
983
|
-
generation_config: z.record(z.string(), z.unknown()).optional(),
|
|
984
|
-
log_uri: z.string().optional()
|
|
985
|
-
}).strict();
|
|
986
|
-
var comparisonReportSchema = z.object({
|
|
987
|
-
avg_score_delta: z.number(),
|
|
988
|
-
pass_rate_delta: z.number(),
|
|
989
|
-
exact_match_rate_delta: z.number(),
|
|
990
|
-
token_f1_delta: z.number().optional(),
|
|
991
|
-
regressions: z.number().int().nonnegative(),
|
|
992
|
-
improvements: z.number().int().nonnegative(),
|
|
993
|
-
regressed_examples: z.array(z.object({
|
|
994
|
-
prompt: z.string(),
|
|
995
|
-
old_score: z.number(),
|
|
996
|
-
new_score: z.number()
|
|
997
|
-
}).strict())
|
|
998
|
-
}).strict();
|
|
999
|
-
var trainingReportSchema = z.object({
|
|
1000
|
-
provider: z.literal("local-uv"),
|
|
1001
|
-
training_job_name: z.string(),
|
|
1002
|
-
model_artifact_uri: z.string().optional(),
|
|
1003
|
-
base_model_artifact_uri: z.string().optional(),
|
|
1004
|
-
metrics: z.record(z.string(), z.unknown()).nullable(),
|
|
1005
|
-
exit_code: z.number().int().nullable(),
|
|
1006
|
-
log_uri: z.string(),
|
|
1007
|
-
command: z.array(z.string()).optional()
|
|
1008
|
-
}).strict();
|
|
1009
|
-
var runReportSchema = z.object({
|
|
1010
|
-
run_id: z.string().uuid(),
|
|
1011
|
-
behavior_spec_id: z.string().uuid(),
|
|
1012
|
-
user_id: z.string(),
|
|
1013
|
-
run_number: z.number().int().min(1),
|
|
1014
|
-
base_model: z.string(),
|
|
1015
|
-
fine_tuned_model_id: z.string(),
|
|
1016
|
-
status: z.enum(["completed", "failed"]),
|
|
1017
|
-
baseline: evalReportSchema,
|
|
1018
|
-
candidate: evalReportSchema,
|
|
1019
|
-
comparison: comparisonReportSchema,
|
|
1020
|
-
general_regression: z.object({
|
|
1021
|
-
dataset_uri: z.string(),
|
|
1022
|
-
dataset_sha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
1023
|
-
baseline: evalReportSchema,
|
|
1024
|
-
candidate: evalReportSchema,
|
|
1025
|
-
comparison: comparisonReportSchema,
|
|
1026
|
-
policy: z.object({
|
|
1027
|
-
max_score_drop: z.number().min(0).max(1),
|
|
1028
|
-
max_pass_rate_drop: z.number().min(0).max(1)
|
|
1029
|
-
}).strict(),
|
|
1030
|
-
passed: z.boolean(),
|
|
1031
|
-
failures: z.array(z.string())
|
|
1032
|
-
}).strict().optional(),
|
|
1033
|
-
training: trainingReportSchema,
|
|
1034
|
-
artifact_uris: z.object({
|
|
1035
|
-
dataset: z.string(),
|
|
1036
|
-
baseline_eval: z.string(),
|
|
1037
|
-
candidate_eval: z.string(),
|
|
1038
|
-
general_baseline_eval: z.string().optional(),
|
|
1039
|
-
general_candidate_eval: z.string().optional(),
|
|
1040
|
-
report: z.string()
|
|
1041
|
-
}).strict(),
|
|
1042
|
-
run_metadata: z.object({
|
|
1043
|
-
base_model: z.string(),
|
|
1044
|
-
fine_tuned_model_id: z.string(),
|
|
1045
|
-
dataset_prebuilt: z.boolean(),
|
|
1046
|
-
dataset_format: datasetFormatSchema.nullable().optional(),
|
|
1047
|
-
dataset_uri: z.string(),
|
|
1048
|
-
spec_example_count: z.number().int().nonnegative(),
|
|
1049
|
-
training_example_count: z.number().int().nonnegative().nullable(),
|
|
1050
|
-
eval_examples_total: z.number().int().nonnegative(),
|
|
1051
|
-
eval_examples_used: z.number().int().nonnegative(),
|
|
1052
|
-
eval_split: evalSplitSchema.optional(),
|
|
1053
|
-
eval_sample_seed: z.number().int().nullable().optional(),
|
|
1054
|
-
started_at: z.string(),
|
|
1055
|
-
completed_at: z.string(),
|
|
1056
|
-
elapsed_ms: z.number().int().nonnegative(),
|
|
1057
|
-
elapsed_seconds: z.number().nonnegative()
|
|
1058
|
-
}).strict(),
|
|
1059
|
-
created_at: z.string()
|
|
1060
|
-
}).strict();
|
|
1061
|
-
var inferenceConfigSchema = z.object({
|
|
1062
|
-
maxNewTokens: z.number().int().min(1).default(256),
|
|
1063
|
-
temperature: z.number().min(0).default(0),
|
|
1064
|
-
topP: z.number().min(0).max(1).default(1),
|
|
1065
|
-
device: z.enum(["cuda", "cpu"]).default("cuda")
|
|
1066
|
-
}).strict();
|
|
1067
|
-
var scoringConfigSchema = z.object({
|
|
1068
|
-
mode: z.enum(["exact_match", "json_fields"]).default("exact_match"),
|
|
1069
|
-
fields: z.array(z.string().min(1)).optional()
|
|
1070
|
-
}).strict();
|
|
1071
|
-
var evaluationConfigSchema = z.object({
|
|
1072
|
-
inference: inferenceConfigSchema.default({
|
|
1073
|
-
maxNewTokens: 256,
|
|
1074
|
-
temperature: 0,
|
|
1075
|
-
topP: 1,
|
|
1076
|
-
device: "cuda"
|
|
1077
|
-
}),
|
|
1078
|
-
scoring: scoringConfigSchema.default({ mode: "exact_match" }),
|
|
1079
|
-
timeoutMs: z.number().int().min(100).default(18e5),
|
|
1080
|
-
maxExamples: z.number().int().min(1).optional(),
|
|
1081
|
-
sampleSeed: z.number().int().optional(),
|
|
1082
|
-
baselineCache: z.boolean().default(true),
|
|
1083
|
-
generalRegression: z.object({
|
|
1084
|
-
dataset: z.string().min(1),
|
|
1085
|
-
systemPrompt: z.string().optional(),
|
|
1086
|
-
maxScoreDrop: z.number().min(0).max(1).default(0.03),
|
|
1087
|
-
maxPassRateDrop: z.number().min(0).max(1).default(0.05)
|
|
1088
|
-
}).strict().optional()
|
|
1089
|
-
}).strict();
|
|
1090
|
-
var localRunnerConfigSchema = z.object({
|
|
1091
|
-
storeRoot: z.string().optional(),
|
|
1092
|
-
artifactRoot: z.string().default(".tt-local/artifacts"),
|
|
1093
|
-
dryRun: z.boolean().default(false),
|
|
1094
|
-
paths: z.object({
|
|
1095
|
-
baseModel: z.string().optional(),
|
|
1096
|
-
modelCache: z.string().optional()
|
|
1097
|
-
}).strict().default({}),
|
|
1098
|
-
evaluation: evaluationConfigSchema.default({
|
|
1099
|
-
inference: {
|
|
1100
|
-
maxNewTokens: 256,
|
|
1101
|
-
temperature: 0,
|
|
1102
|
-
topP: 1,
|
|
1103
|
-
device: "cuda"
|
|
1104
|
-
},
|
|
1105
|
-
scoring: { mode: "exact_match" },
|
|
1106
|
-
timeoutMs: 18e5,
|
|
1107
|
-
baselineCache: true
|
|
1108
|
-
})
|
|
1109
|
-
}).strict();
|
|
1110
|
-
function isFoundationSpecFile(spec) {
|
|
1111
|
-
return spec.engine === "foundation";
|
|
879
|
+
return root;
|
|
1112
880
|
}
|
|
1113
881
|
|
|
1114
|
-
// src/local-runtime/
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
882
|
+
// src/local-runtime/capability-profiles.ts
|
|
883
|
+
var GIB = 1024 ** 3;
|
|
884
|
+
function gib(n) {
|
|
885
|
+
return Math.round(n * GIB);
|
|
886
|
+
}
|
|
887
|
+
var FOUNDATION_DEFAULT_DEPTH = 2;
|
|
888
|
+
var FOUNDATION_MAX_DEPTH = 64;
|
|
889
|
+
var FOUNDATION_DEFAULT_VOCAB = 256;
|
|
890
|
+
var FOUNDATION_DEFAULT_SEQUENCE = 64;
|
|
891
|
+
var FOUNDATION_DEFAULT_BATCH = 2;
|
|
892
|
+
var ADAPTER_MEMORY_PROFILES = [
|
|
893
|
+
{
|
|
894
|
+
id: "Qwen/Qwen3.5-2B",
|
|
895
|
+
diskBytes: gib(8),
|
|
896
|
+
inferenceBytes: gib(8),
|
|
897
|
+
trainBytes: gib(16),
|
|
898
|
+
sparkClass: false
|
|
899
|
+
},
|
|
900
|
+
{
|
|
901
|
+
id: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
|
|
902
|
+
diskBytes: gib(80),
|
|
903
|
+
inferenceBytes: gib(80),
|
|
904
|
+
trainBytes: gib(96),
|
|
905
|
+
sparkClass: true
|
|
906
|
+
},
|
|
907
|
+
{
|
|
908
|
+
id: "meta-models/Muse-Glimmer-30B",
|
|
909
|
+
diskBytes: gib(80),
|
|
910
|
+
inferenceBytes: gib(80),
|
|
911
|
+
trainBytes: gib(96),
|
|
912
|
+
sparkClass: true
|
|
913
|
+
}
|
|
914
|
+
];
|
|
915
|
+
function adapterMemoryProfile(modelId) {
|
|
916
|
+
const normalized = modelId.trim().toLowerCase();
|
|
917
|
+
return ADAPTER_MEMORY_PROFILES.find((profile) => profile.id.toLowerCase() === normalized);
|
|
918
|
+
}
|
|
919
|
+
function foundationWidth(depth) {
|
|
920
|
+
return 32 * depth;
|
|
921
|
+
}
|
|
922
|
+
function foundationParamCount(depth, vocabSize = FOUNDATION_DEFAULT_VOCAB, sequenceLength = FOUNDATION_DEFAULT_SEQUENCE) {
|
|
923
|
+
const width = foundationWidth(depth);
|
|
924
|
+
return 12 * depth * width * width + vocabSize * width + sequenceLength * width;
|
|
925
|
+
}
|
|
926
|
+
function foundationTrainBytes(depth, options = {}) {
|
|
927
|
+
const vocabSize = options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB;
|
|
928
|
+
const sequenceLength = options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE;
|
|
929
|
+
const batchSize = options.batchSize ?? FOUNDATION_DEFAULT_BATCH;
|
|
930
|
+
const params = foundationParamCount(depth, vocabSize, sequenceLength);
|
|
931
|
+
const width = foundationWidth(depth);
|
|
932
|
+
const activationSlack = batchSize * sequenceLength * width * depth * 64;
|
|
933
|
+
return 16 * params + activationSlack;
|
|
934
|
+
}
|
|
935
|
+
function foundationInferenceBytes(depth, options = {}) {
|
|
936
|
+
const params = foundationParamCount(
|
|
937
|
+
depth,
|
|
938
|
+
options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB,
|
|
939
|
+
options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE
|
|
940
|
+
);
|
|
941
|
+
return 4 * params + gib(0.25);
|
|
942
|
+
}
|
|
943
|
+
function isUnifiedGpuName(name) {
|
|
944
|
+
if (!name) return false;
|
|
945
|
+
return /spark|gb10/i.test(name);
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
// src/local-runtime/capability.ts
|
|
949
|
+
var HEADROOM = 1.2;
|
|
950
|
+
var FIT_RATIO = 0.7;
|
|
951
|
+
function formatGiB(bytes) {
|
|
952
|
+
const amount = bytes / 1024 ** 3;
|
|
953
|
+
const precision = amount >= 10 ? 0 : 1;
|
|
954
|
+
return `${amount.toFixed(precision)} GiB`;
|
|
955
|
+
}
|
|
956
|
+
function compareFit(available, required, missing) {
|
|
957
|
+
if (available === void 0) {
|
|
958
|
+
return { status: "not_possible", reason: missing, required_bytes: required };
|
|
959
|
+
}
|
|
960
|
+
if (available < required) {
|
|
961
|
+
return {
|
|
962
|
+
status: "not_possible",
|
|
963
|
+
reason: `needs ~${formatGiB(required)}; ${formatGiB(available)} available`,
|
|
964
|
+
required_bytes: required,
|
|
965
|
+
available_bytes: available
|
|
966
|
+
};
|
|
967
|
+
}
|
|
968
|
+
if (available < required * HEADROOM) {
|
|
969
|
+
return {
|
|
970
|
+
status: "tight",
|
|
971
|
+
reason: `fits in ${formatGiB(available)} with little headroom (needs ~${formatGiB(required)})`,
|
|
972
|
+
required_bytes: required,
|
|
973
|
+
available_bytes: available
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
return {
|
|
977
|
+
status: "ready",
|
|
978
|
+
reason: `${formatGiB(available)} available (needs ~${formatGiB(required)})`,
|
|
979
|
+
required_bytes: required,
|
|
980
|
+
available_bytes: available
|
|
981
|
+
};
|
|
982
|
+
}
|
|
983
|
+
function trainingBlockers(inventory) {
|
|
984
|
+
const blockers = [];
|
|
985
|
+
if (!inventory.node.ok) blockers.push(`Node ${inventory.node.version} is below 22`);
|
|
986
|
+
if (inventory.uv && !inventory.uv.ok) blockers.push(inventory.uv.message);
|
|
987
|
+
if (inventory.python && !inventory.python.ok) blockers.push(inventory.python.message);
|
|
988
|
+
return blockers;
|
|
989
|
+
}
|
|
990
|
+
function cudaAvailable(inventory) {
|
|
991
|
+
if (inventory.python?.ok && inventory.python.cuda_available !== void 0) {
|
|
992
|
+
return inventory.python.cuda_available;
|
|
993
|
+
}
|
|
994
|
+
return inventory.nvidia_smi.ok && inventory.gpus.length > 0;
|
|
995
|
+
}
|
|
996
|
+
function trainingGpu(inventory) {
|
|
997
|
+
const listed = inventory.gpus.find((gpu) => gpu.index === 0) ?? inventory.gpus[0];
|
|
998
|
+
if (listed) {
|
|
999
|
+
if (!listed.memory_total_bytes && inventory.python?.total_memory_bytes) {
|
|
1000
|
+
return { ...listed, memory_total_bytes: inventory.python.total_memory_bytes };
|
|
1001
|
+
}
|
|
1002
|
+
return listed;
|
|
1003
|
+
}
|
|
1004
|
+
if (inventory.python?.cuda_device) {
|
|
1005
|
+
return {
|
|
1006
|
+
index: 0,
|
|
1007
|
+
name: inventory.python.cuda_device,
|
|
1008
|
+
memory_total_bytes: inventory.python.total_memory_bytes,
|
|
1009
|
+
unified_memory: isUnifiedGpuName(inventory.python.cuda_device)
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
return void 0;
|
|
1013
|
+
}
|
|
1014
|
+
function gpuMemory(gpu) {
|
|
1015
|
+
return gpu?.memory_total_bytes;
|
|
1016
|
+
}
|
|
1017
|
+
function diskForCache(inventory) {
|
|
1018
|
+
return inventory.disks.find((disk) => disk.name === "model-cache")?.free_bytes;
|
|
1019
|
+
}
|
|
1020
|
+
function worse(left, right) {
|
|
1021
|
+
const rank = { not_possible: 2, tight: 1, ready: 0 };
|
|
1022
|
+
return rank[left.status] >= rank[right.status] ? left : right;
|
|
1023
|
+
}
|
|
1024
|
+
function withBlockers(verdict, blockers) {
|
|
1025
|
+
if (blockers.length === 0) return verdict;
|
|
1026
|
+
return {
|
|
1027
|
+
status: "not_possible",
|
|
1028
|
+
reason: blockers.join("; "),
|
|
1029
|
+
required_bytes: verdict.required_bytes,
|
|
1030
|
+
available_bytes: verdict.available_bytes
|
|
1031
|
+
};
|
|
1032
|
+
}
|
|
1033
|
+
function adapterVerdicts(profile, inventory, gpu, hasCuda) {
|
|
1034
|
+
const blockers = trainingBlockers(inventory);
|
|
1035
|
+
const gpuBytes = gpuMemory(gpu);
|
|
1036
|
+
const ram = inventory.os.total_memory_bytes;
|
|
1037
|
+
const disk = diskForCache(inventory);
|
|
1038
|
+
let train;
|
|
1039
|
+
if (!hasCuda) {
|
|
1040
|
+
train = {
|
|
1041
|
+
status: "not_possible",
|
|
1042
|
+
reason: "LoRA training requires CUDA",
|
|
1043
|
+
required_bytes: profile.trainBytes
|
|
1044
|
+
};
|
|
1045
|
+
} else {
|
|
1046
|
+
train = compareFit(gpuBytes, profile.trainBytes, "GPU memory is unknown");
|
|
1047
|
+
}
|
|
1048
|
+
train = withBlockers(train, blockers);
|
|
1049
|
+
if (disk !== void 0 && disk < profile.diskBytes && train.status !== "not_possible") {
|
|
1050
|
+
train = worse(train, compareFit(disk, profile.diskBytes, "model cache disk is unknown"));
|
|
1051
|
+
}
|
|
1052
|
+
const finetune = train;
|
|
1053
|
+
let inference;
|
|
1054
|
+
if (hasCuda) {
|
|
1055
|
+
inference = compareFit(gpuBytes, profile.inferenceBytes, "GPU memory is unknown");
|
|
1056
|
+
} else {
|
|
1057
|
+
inference = compareFit(ram, profile.inferenceBytes, "system RAM is unknown");
|
|
1058
|
+
if (inference.status !== "not_possible") {
|
|
1059
|
+
inference = {
|
|
1060
|
+
...inference,
|
|
1061
|
+
reason: `CPU inference: ${inference.reason}`
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1064
|
+
}
|
|
1065
|
+
return {
|
|
1066
|
+
id: profile.id,
|
|
1067
|
+
spark_class: profile.sparkClass,
|
|
1068
|
+
train,
|
|
1069
|
+
finetune,
|
|
1070
|
+
inference
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
function suggestedFoundationDepth(availableBytes) {
|
|
1074
|
+
if (availableBytes === void 0) return FOUNDATION_DEFAULT_DEPTH;
|
|
1075
|
+
const budget = availableBytes * FIT_RATIO;
|
|
1076
|
+
let suggested = FOUNDATION_DEFAULT_DEPTH;
|
|
1077
|
+
for (let depth = FOUNDATION_DEFAULT_DEPTH; depth <= FOUNDATION_MAX_DEPTH; depth += 1) {
|
|
1078
|
+
if (foundationTrainBytes(depth) <= budget) suggested = depth;
|
|
1079
|
+
else break;
|
|
1080
|
+
}
|
|
1081
|
+
return suggested;
|
|
1082
|
+
}
|
|
1083
|
+
function evaluateCapabilities(inventory) {
|
|
1084
|
+
const hasCuda = cudaAvailable(inventory);
|
|
1085
|
+
const gpu = trainingGpu(inventory);
|
|
1086
|
+
const notes = [];
|
|
1087
|
+
if (gpu?.unified_memory || isUnifiedGpuName(gpu?.name)) {
|
|
1088
|
+
notes.push("GPU memory is a unified pool shared with system RAM (Spark-class).");
|
|
1089
|
+
}
|
|
1090
|
+
if (inventory.quick) {
|
|
1091
|
+
notes.push("Quick probe skipped the bundled torch/uv runtime.");
|
|
1092
|
+
}
|
|
1093
|
+
if (!inventory.node.ok) {
|
|
1094
|
+
notes.push(`Node ${inventory.node.version} is below the 22.x required by the local runner.`);
|
|
1095
|
+
}
|
|
1096
|
+
const adapters = ADAPTER_MEMORY_PROFILES.map(
|
|
1097
|
+
(profile) => adapterVerdicts(profile, inventory, gpu, hasCuda)
|
|
1098
|
+
);
|
|
1099
|
+
const blockers = trainingBlockers(inventory);
|
|
1100
|
+
const gpuBytes = gpuMemory(gpu);
|
|
1101
|
+
const ram = inventory.os.total_memory_bytes;
|
|
1102
|
+
const trainBudget = hasCuda ? gpuBytes : void 0;
|
|
1103
|
+
const suggested = suggestedFoundationDepth(trainBudget);
|
|
1104
|
+
const defaultTrain = foundationTrainBytes(FOUNDATION_DEFAULT_DEPTH);
|
|
1105
|
+
const inferBudget = hasCuda ? gpuBytes : ram;
|
|
1106
|
+
let train;
|
|
1107
|
+
if (!hasCuda) {
|
|
1108
|
+
train = {
|
|
1109
|
+
status: "not_possible",
|
|
1110
|
+
reason: "Foundation pretrain, SFT, and RL require CUDA",
|
|
1111
|
+
required_bytes: defaultTrain
|
|
1112
|
+
};
|
|
1113
|
+
} else {
|
|
1114
|
+
train = compareFit(gpuBytes, defaultTrain, "GPU memory is unknown");
|
|
1115
|
+
}
|
|
1116
|
+
train = withBlockers(train, blockers);
|
|
1117
|
+
const inference = compareFit(
|
|
1118
|
+
inferBudget,
|
|
1119
|
+
foundationInferenceBytes(FOUNDATION_DEFAULT_DEPTH),
|
|
1120
|
+
hasCuda ? "GPU memory is unknown" : "system RAM is unknown"
|
|
1121
|
+
);
|
|
1122
|
+
const foundation = {
|
|
1123
|
+
default_depth: FOUNDATION_DEFAULT_DEPTH,
|
|
1124
|
+
suggested_max_depth: hasCuda && blockers.length === 0 ? suggested : FOUNDATION_DEFAULT_DEPTH,
|
|
1125
|
+
train,
|
|
1126
|
+
finetune: train,
|
|
1127
|
+
inference: hasCuda ? inference : { ...inference, reason: `CPU eval: ${inference.reason}` },
|
|
1128
|
+
serve: {
|
|
1129
|
+
status: "not_possible",
|
|
1130
|
+
reason: "tt serve cannot host foundation checkpoints yet"
|
|
1131
|
+
}
|
|
1132
|
+
};
|
|
1133
|
+
return { cuda_available: hasCuda, gpu, adapters, foundation, notes };
|
|
1134
|
+
}
|
|
1135
|
+
function formatCapabilitySummary(report) {
|
|
1136
|
+
const gpu = report.gpu;
|
|
1137
|
+
const gpuLabel = gpu ? `${gpu.name}${gpu.memory_total_bytes ? `, ${formatGiB(gpu.memory_total_bytes)}` : ""}${gpu.unified_memory ? " unified" : ""}` : "no NVIDIA GPU";
|
|
1138
|
+
const adapterBits = report.adapters.map((adapter) => {
|
|
1139
|
+
const short = adapter.id.split("/").at(-1) ?? adapter.id;
|
|
1140
|
+
return `${short} train ${adapter.train.status}`;
|
|
1141
|
+
});
|
|
1142
|
+
return [
|
|
1143
|
+
`GPU ${gpuLabel}, CUDA ${report.cuda_available ? "yes" : "no"}`,
|
|
1144
|
+
`Adapter: ${adapterBits.join("; ")}`,
|
|
1145
|
+
`Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve not supported`
|
|
1146
|
+
].join(". ");
|
|
1147
|
+
}
|
|
1148
|
+
function warningsFromSnapshot(report, target) {
|
|
1149
|
+
const warnings = [];
|
|
1150
|
+
if (target.engine === "foundation") {
|
|
1151
|
+
for (const kind of ["train", "finetune"]) {
|
|
1152
|
+
const verdict = report.foundation[kind];
|
|
1153
|
+
if (verdict.status === "not_possible" || verdict.status === "tight") {
|
|
1154
|
+
warnings.push(`Foundation ${kind} is ${verdict.status}: ${verdict.reason}`);
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
return warnings;
|
|
1158
|
+
}
|
|
1159
|
+
const modelId = target.baseModel;
|
|
1160
|
+
const adapter = modelId ? report.adapters.find((item) => item.id.toLowerCase() === modelId.trim().toLowerCase()) : void 0;
|
|
1161
|
+
if (!adapter) {
|
|
1162
|
+
if (modelId && !adapterMemoryProfile(modelId)) return warnings;
|
|
1163
|
+
const blocked = report.adapters.filter((item) => item.train.status === "not_possible");
|
|
1164
|
+
if (blocked.length === report.adapters.length) {
|
|
1165
|
+
warnings.push(`Adapter train is not_possible: ${report.adapters[0]?.train.reason ?? "CUDA required"}`);
|
|
1166
|
+
}
|
|
1167
|
+
return warnings;
|
|
1168
|
+
}
|
|
1169
|
+
for (const kind of ["train", "finetune", "inference"]) {
|
|
1170
|
+
const verdict = adapter[kind];
|
|
1171
|
+
if (verdict.status === "not_possible" || verdict.status === "tight") {
|
|
1172
|
+
warnings.push(`${adapter.id} ${kind} is ${verdict.status}: ${verdict.reason}`);
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
return warnings;
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
// src/local-runtime/host-inventory.ts
|
|
1179
|
+
import { constants } from "fs";
|
|
1180
|
+
import { access, statfs } from "fs/promises";
|
|
1181
|
+
import { cpus, freemem, homedir as homedir3, totalmem, type } from "os";
|
|
1182
|
+
import { dirname as dirname5, join as join5, resolve as resolve7 } from "path";
|
|
1183
|
+
import { spawn as spawn2 } from "child_process";
|
|
1184
|
+
|
|
1185
|
+
// src/paths.ts
|
|
1186
|
+
import { existsSync as existsSync2 } from "fs";
|
|
1118
1187
|
import { homedir } from "os";
|
|
1188
|
+
import { isAbsolute as isAbsolute2, join as join2, resolve as resolve4 } from "path";
|
|
1189
|
+
var TUNED_TENSOR_DIR = ".tuned-tensor";
|
|
1190
|
+
var DEFAULT_ARTIFACT_ROOT = `${TUNED_TENSOR_DIR}/artifacts`;
|
|
1191
|
+
var DEFAULT_PROJECT_STORE_ROOT = `${TUNED_TENSOR_DIR}/store`;
|
|
1192
|
+
var DEFAULT_FOUNDATION_RUNS_DIR = `${TUNED_TENSOR_DIR}/foundation-runs`;
|
|
1193
|
+
function envHome(env = process.env) {
|
|
1194
|
+
const home = env.HOME?.trim();
|
|
1195
|
+
return home ? resolve4(home) : homedir();
|
|
1196
|
+
}
|
|
1197
|
+
function expandUserPath(value, env = process.env, baseDirectory = process.cwd()) {
|
|
1198
|
+
const trimmed = value.trim();
|
|
1199
|
+
if (trimmed === "~") return envHome(env);
|
|
1200
|
+
if (trimmed.startsWith("~/")) return resolve4(envHome(env), trimmed.slice(2));
|
|
1201
|
+
return isAbsolute2(trimmed) ? resolve4(trimmed) : resolve4(baseDirectory, trimmed);
|
|
1202
|
+
}
|
|
1203
|
+
function getTunedTensorHome(env = process.env) {
|
|
1204
|
+
const override = env.TUNED_TENSOR_HOME?.trim();
|
|
1205
|
+
if (override) return expandUserPath(override, env);
|
|
1206
|
+
return join2(envHome(env), TUNED_TENSOR_DIR);
|
|
1207
|
+
}
|
|
1208
|
+
function legacyConfigDir(env) {
|
|
1209
|
+
const configHome = env.XDG_CONFIG_HOME?.trim() ? resolve4(env.XDG_CONFIG_HOME) : join2(envHome(env), ".config");
|
|
1210
|
+
return join2(configHome, "tuned-tensor");
|
|
1211
|
+
}
|
|
1212
|
+
function getConfigDir(env = process.env) {
|
|
1213
|
+
const next = getTunedTensorHome(env);
|
|
1214
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
1215
|
+
const legacy = legacyConfigDir(env);
|
|
1216
|
+
if (!existsSync2(join2(next, "config.json")) && existsSync2(join2(legacy, "config.json"))) {
|
|
1217
|
+
return legacy;
|
|
1218
|
+
}
|
|
1219
|
+
return next;
|
|
1220
|
+
}
|
|
1221
|
+
function getAgentConfigDir(env = process.env) {
|
|
1222
|
+
const next = join2(getTunedTensorHome(env), "agent");
|
|
1223
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
1224
|
+
const legacy = join2(legacyConfigDir(env), "agent");
|
|
1225
|
+
if (!existsSync2(next) && existsSync2(legacy)) return legacy;
|
|
1226
|
+
return next;
|
|
1227
|
+
}
|
|
1228
|
+
function defaultStoreRoot(env = process.env) {
|
|
1229
|
+
const override = env.TT_LOCAL_HOME?.trim();
|
|
1230
|
+
if (override) return expandUserPath(override, env);
|
|
1231
|
+
const next = join2(getTunedTensorHome(env), "store");
|
|
1232
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
1233
|
+
const legacy = join2(envHome(env), ".tuned-tensor-local");
|
|
1234
|
+
if (!existsSync2(next) && existsSync2(legacy)) return legacy;
|
|
1235
|
+
return next;
|
|
1236
|
+
}
|
|
1237
|
+
function defaultCacheRoot(env = process.env) {
|
|
1238
|
+
return join2(getTunedTensorHome(env), "cache");
|
|
1239
|
+
}
|
|
1240
|
+
function pythonEnvironmentPath(kind, hash, env = process.env) {
|
|
1241
|
+
const next = join2(defaultCacheRoot(env), kind, hash);
|
|
1242
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
1243
|
+
const cacheHome = env.XDG_CACHE_HOME?.trim() ? resolve4(env.XDG_CACHE_HOME) : join2(envHome(env), ".cache");
|
|
1244
|
+
const legacy = join2(cacheHome, "tuned-tensor-local", kind, hash);
|
|
1245
|
+
if (!existsSync2(next) && existsSync2(legacy)) return legacy;
|
|
1246
|
+
return next;
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
// src/local-runtime/huggingface-cache.ts
|
|
1250
|
+
import { homedir as homedir2 } from "os";
|
|
1251
|
+
import { dirname as dirname3, join as join3, resolve as resolve5 } from "path";
|
|
1252
|
+
function minimalMachineLearningEnvironment(env) {
|
|
1253
|
+
const exact = /* @__PURE__ */ new Set([
|
|
1254
|
+
"PATH",
|
|
1255
|
+
"HOME",
|
|
1256
|
+
"USER",
|
|
1257
|
+
"LOGNAME",
|
|
1258
|
+
"SHELL",
|
|
1259
|
+
"TMPDIR",
|
|
1260
|
+
"TEMP",
|
|
1261
|
+
"TMP",
|
|
1262
|
+
"LANG",
|
|
1263
|
+
"LANGUAGE",
|
|
1264
|
+
"LD_LIBRARY_PATH",
|
|
1265
|
+
"DYLD_LIBRARY_PATH",
|
|
1266
|
+
"XDG_CACHE_HOME",
|
|
1267
|
+
"SSL_CERT_FILE",
|
|
1268
|
+
"REQUESTS_CA_BUNDLE",
|
|
1269
|
+
"CURL_CA_BUNDLE",
|
|
1270
|
+
"NO_PROXY",
|
|
1271
|
+
"no_proxy",
|
|
1272
|
+
"UV_CACHE_DIR",
|
|
1273
|
+
"UV_PYTHON",
|
|
1274
|
+
"HF_HOME",
|
|
1275
|
+
"HF_HUB_CACHE",
|
|
1276
|
+
"HUGGINGFACE_HUB_CACHE",
|
|
1277
|
+
"HF_ENDPOINT",
|
|
1278
|
+
"HF_TOKEN",
|
|
1279
|
+
"HF_HUB_OFFLINE",
|
|
1280
|
+
"HF_HUB_DISABLE_TELEMETRY",
|
|
1281
|
+
"TRANSFORMERS_OFFLINE",
|
|
1282
|
+
"HF_HUB_ENABLE_HF_TRANSFER",
|
|
1283
|
+
"PYTHONUTF8",
|
|
1284
|
+
"PYTHONIOENCODING"
|
|
1285
|
+
]);
|
|
1286
|
+
const prefixes = [
|
|
1287
|
+
"LC_",
|
|
1288
|
+
"CUDA_",
|
|
1289
|
+
"NVIDIA_",
|
|
1290
|
+
"NCCL_",
|
|
1291
|
+
"OMP_",
|
|
1292
|
+
"MKL_",
|
|
1293
|
+
"TORCH_",
|
|
1294
|
+
"PYTORCH_",
|
|
1295
|
+
"TOKENIZERS_",
|
|
1296
|
+
"HF_XET_"
|
|
1297
|
+
];
|
|
1298
|
+
const result = Object.fromEntries(Object.entries(env).filter(
|
|
1299
|
+
([key]) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix))
|
|
1300
|
+
));
|
|
1301
|
+
result.TORCH_DISABLE_NATIVE_JIT ??= "1";
|
|
1302
|
+
return result;
|
|
1303
|
+
}
|
|
1304
|
+
function resolveHuggingFaceCacheLayout(modelCache) {
|
|
1305
|
+
const hfHome = resolve5(modelCache);
|
|
1306
|
+
return {
|
|
1307
|
+
hfHome,
|
|
1308
|
+
hubCache: join3(hfHome, "hub")
|
|
1309
|
+
};
|
|
1310
|
+
}
|
|
1311
|
+
function withHuggingFaceCacheEnvironment(env, modelCache) {
|
|
1312
|
+
const inheritedHub = env.HF_HUB_CACHE ?? env.HUGGINGFACE_HUB_CACHE;
|
|
1313
|
+
const canonicalHome = modelCache ?? env.HF_HOME ?? (inheritedHub ? dirname3(resolve5(inheritedHub)) : join3(homedir2(), ".cache", "huggingface"));
|
|
1314
|
+
const layout = resolveHuggingFaceCacheLayout(canonicalHome);
|
|
1315
|
+
const result = {
|
|
1316
|
+
...env,
|
|
1317
|
+
HF_HOME: layout.hfHome,
|
|
1318
|
+
HF_HUB_CACHE: layout.hubCache,
|
|
1319
|
+
HUGGINGFACE_HUB_CACHE: layout.hubCache
|
|
1320
|
+
};
|
|
1321
|
+
delete result.TRANSFORMERS_CACHE;
|
|
1322
|
+
delete result.PYTORCH_TRANSFORMERS_CACHE;
|
|
1323
|
+
delete result.PYTORCH_PRETRAINED_BERT_CACHE;
|
|
1324
|
+
return result;
|
|
1325
|
+
}
|
|
1326
|
+
function withOfflineHuggingFaceCacheEnvironment(env, modelCache) {
|
|
1327
|
+
return {
|
|
1328
|
+
...withHuggingFaceCacheEnvironment(env, modelCache),
|
|
1329
|
+
HF_HUB_OFFLINE: "1",
|
|
1330
|
+
TRANSFORMERS_OFFLINE: "1"
|
|
1331
|
+
};
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
// src/local-runtime/process-runner.ts
|
|
1335
|
+
import { createHash as createHash2 } from "crypto";
|
|
1336
|
+
import { spawn } from "child_process";
|
|
1337
|
+
import { createWriteStream, readFileSync } from "fs";
|
|
1119
1338
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1339
|
+
import { dirname as dirname4, join as join4, resolve as resolve6 } from "path";
|
|
1340
|
+
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
1341
|
+
var bundledProject = join4(packageRoot, "training/adapter");
|
|
1342
|
+
var bundledRuntimeHash = (() => {
|
|
1343
|
+
const hash = createHash2("sha256");
|
|
1344
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1345
|
+
hash.update(readFileSync(join4(bundledProject, name)));
|
|
1124
1346
|
}
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
const
|
|
1132
|
-
|
|
1133
|
-
const dataset = value.dataset_prebuilt;
|
|
1134
|
-
if (dataset && typeof dataset === "object" && !Array.isArray(dataset)) {
|
|
1135
|
-
const fields = dataset;
|
|
1136
|
-
for (const key of ["training", "validation", "test"]) {
|
|
1137
|
-
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
1138
|
-
}
|
|
1347
|
+
return hash.digest("hex").slice(0, 20);
|
|
1348
|
+
})();
|
|
1349
|
+
var BUNDLED_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv", bundledRuntimeHash);
|
|
1350
|
+
var foundationProject = join4(packageRoot, "training/foundation");
|
|
1351
|
+
var foundationRuntimeHash = (() => {
|
|
1352
|
+
const hash = createHash2("sha256");
|
|
1353
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1354
|
+
hash.update(readFileSync(join4(foundationProject, name)));
|
|
1139
1355
|
}
|
|
1140
|
-
return
|
|
1356
|
+
return hash.digest("hex").slice(0, 20);
|
|
1357
|
+
})();
|
|
1358
|
+
var FOUNDATION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-foundation", foundationRuntimeHash);
|
|
1359
|
+
function buildBundledPythonCommand(entrypoint, args = []) {
|
|
1360
|
+
const target = entrypoint === "-c" ? entrypoint : join4(bundledProject, "src", entrypoint);
|
|
1361
|
+
const commandArgs = [
|
|
1362
|
+
"run",
|
|
1363
|
+
"--frozen",
|
|
1364
|
+
"--quiet",
|
|
1365
|
+
"--project",
|
|
1366
|
+
bundledProject,
|
|
1367
|
+
"python",
|
|
1368
|
+
target,
|
|
1369
|
+
...args
|
|
1370
|
+
];
|
|
1371
|
+
return {
|
|
1372
|
+
command: "uv",
|
|
1373
|
+
commandArgs,
|
|
1374
|
+
displayCommand: ["uv", ...commandArgs]
|
|
1375
|
+
};
|
|
1141
1376
|
}
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
}
|
|
1147
|
-
return false;
|
|
1148
|
-
}
|
|
1377
|
+
function withBundledPythonEnvironment(env) {
|
|
1378
|
+
return {
|
|
1379
|
+
...env,
|
|
1380
|
+
UV_PROJECT_ENVIRONMENT: BUNDLED_PYTHON_ENVIRONMENT
|
|
1381
|
+
};
|
|
1149
1382
|
}
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1383
|
+
function buildFoundationPythonCommand(entrypoint, args = []) {
|
|
1384
|
+
const commandArgs = [
|
|
1385
|
+
"run",
|
|
1386
|
+
"--frozen",
|
|
1387
|
+
"--quiet",
|
|
1388
|
+
"--project",
|
|
1389
|
+
foundationProject,
|
|
1390
|
+
"python",
|
|
1391
|
+
join4(foundationProject, "src", entrypoint),
|
|
1392
|
+
...args
|
|
1393
|
+
];
|
|
1394
|
+
return {
|
|
1395
|
+
command: "uv",
|
|
1396
|
+
commandArgs,
|
|
1397
|
+
displayCommand: ["uv", ...commandArgs]
|
|
1398
|
+
};
|
|
1399
|
+
}
|
|
1400
|
+
function withFoundationPythonEnvironment(env) {
|
|
1401
|
+
return {
|
|
1402
|
+
...env,
|
|
1403
|
+
UV_PROJECT_ENVIRONMENT: FOUNDATION_PYTHON_ENVIRONMENT
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1406
|
+
var ProcessCancelledError = class extends Error {
|
|
1407
|
+
constructor(message = "Process cancelled.") {
|
|
1408
|
+
super(message);
|
|
1409
|
+
this.name = "ProcessCancelledError";
|
|
1153
1410
|
}
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1411
|
+
};
|
|
1412
|
+
async function openProcessLog(path, exclusive) {
|
|
1413
|
+
const stream = createWriteStream(path, {
|
|
1414
|
+
flags: exclusive ? "wx" : "w",
|
|
1415
|
+
mode: 384
|
|
1416
|
+
});
|
|
1417
|
+
await new Promise((resolveOpen, reject) => {
|
|
1418
|
+
const onOpen = () => {
|
|
1419
|
+
stream.off("error", onError);
|
|
1420
|
+
resolveOpen();
|
|
1421
|
+
};
|
|
1422
|
+
const onError = (error) => {
|
|
1423
|
+
stream.off("open", onOpen);
|
|
1424
|
+
reject(error);
|
|
1425
|
+
};
|
|
1426
|
+
stream.once("open", onOpen);
|
|
1427
|
+
stream.once("error", onError);
|
|
1428
|
+
});
|
|
1429
|
+
return stream;
|
|
1430
|
+
}
|
|
1431
|
+
async function runLoggedProcess(args) {
|
|
1432
|
+
if (args.logPath) await mkdir2(dirname4(args.logPath), { recursive: true });
|
|
1433
|
+
const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false) : null;
|
|
1434
|
+
let stderr = "";
|
|
1435
|
+
try {
|
|
1436
|
+
const exitCode = await new Promise((resolvePromise, reject) => {
|
|
1437
|
+
const child = spawn(args.command, args.commandArgs, {
|
|
1438
|
+
cwd: args.cwd ? resolve6(args.cwd) : process.cwd(),
|
|
1439
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1440
|
+
env: args.env ?? process.env,
|
|
1441
|
+
detached: process.platform !== "win32"
|
|
1442
|
+
});
|
|
1443
|
+
let timedOut = false;
|
|
1444
|
+
let cancelled = false;
|
|
1445
|
+
let cancellationError;
|
|
1446
|
+
let cancellationCheckRunning = false;
|
|
1447
|
+
let forceKillTimer = null;
|
|
1448
|
+
const killProcessGroup = (signal) => {
|
|
1449
|
+
if (child.pid && process.platform !== "win32") {
|
|
1450
|
+
try {
|
|
1451
|
+
process.kill(-child.pid, signal);
|
|
1452
|
+
return;
|
|
1453
|
+
} catch {
|
|
1454
|
+
}
|
|
1173
1455
|
}
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1456
|
+
child.kill(signal);
|
|
1457
|
+
};
|
|
1458
|
+
const requestStop = (signal = "SIGTERM") => {
|
|
1459
|
+
killProcessGroup(signal);
|
|
1460
|
+
if (!forceKillTimer) {
|
|
1461
|
+
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), 5e3);
|
|
1462
|
+
forceKillTimer.unref();
|
|
1463
|
+
}
|
|
1464
|
+
};
|
|
1465
|
+
const onSigint = () => {
|
|
1466
|
+
cancelled = true;
|
|
1467
|
+
requestStop("SIGINT");
|
|
1468
|
+
};
|
|
1469
|
+
const onSigterm = () => {
|
|
1470
|
+
cancelled = true;
|
|
1471
|
+
requestStop("SIGTERM");
|
|
1472
|
+
};
|
|
1473
|
+
process.once("SIGINT", onSigint);
|
|
1474
|
+
process.once("SIGTERM", onSigterm);
|
|
1475
|
+
const timer = args.timeoutMs ? setTimeout(() => {
|
|
1476
|
+
timedOut = true;
|
|
1477
|
+
requestStop();
|
|
1478
|
+
}, args.timeoutMs) : null;
|
|
1479
|
+
const cancellationTimer = args.shouldCancel ? setInterval(() => {
|
|
1480
|
+
if (cancellationCheckRunning || cancelled) return;
|
|
1481
|
+
cancellationCheckRunning = true;
|
|
1482
|
+
Promise.resolve(args.shouldCancel?.()).then((requested) => {
|
|
1483
|
+
if (!requested || cancelled) return;
|
|
1484
|
+
cancelled = true;
|
|
1485
|
+
requestStop();
|
|
1486
|
+
}).catch((error) => {
|
|
1487
|
+
cancellationError = error;
|
|
1488
|
+
requestStop();
|
|
1489
|
+
}).finally(() => {
|
|
1490
|
+
cancellationCheckRunning = false;
|
|
1491
|
+
});
|
|
1492
|
+
}, args.cancelPollMs ?? 250) : null;
|
|
1493
|
+
cancellationTimer?.unref();
|
|
1494
|
+
const clearProcessTimers = () => {
|
|
1495
|
+
if (timer) clearTimeout(timer);
|
|
1496
|
+
if (cancellationTimer) clearInterval(cancellationTimer);
|
|
1497
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
1498
|
+
process.off("SIGINT", onSigint);
|
|
1499
|
+
process.off("SIGTERM", onSigterm);
|
|
1500
|
+
};
|
|
1501
|
+
if (logStream) {
|
|
1502
|
+
child.stdout.pipe(logStream, { end: false });
|
|
1503
|
+
child.stderr.pipe(logStream, { end: false });
|
|
1213
1504
|
}
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1505
|
+
forwardStreamLines(child.stdout, (line) => {
|
|
1506
|
+
args.onLine?.(line, "stdout");
|
|
1507
|
+
if (args.reporter?.verbose) {
|
|
1508
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stdout", message: line }));
|
|
1509
|
+
}
|
|
1510
|
+
});
|
|
1511
|
+
forwardStreamLines(child.stderr, (line) => {
|
|
1512
|
+
stderr += `${line}
|
|
1513
|
+
`;
|
|
1514
|
+
args.onLine?.(line, "stderr");
|
|
1515
|
+
if (args.reporter?.verbose) {
|
|
1516
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stderr", message: line }));
|
|
1517
|
+
}
|
|
1518
|
+
});
|
|
1519
|
+
child.on("error", (error) => {
|
|
1520
|
+
clearProcessTimers();
|
|
1521
|
+
reject(timedOut ? new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`) : error);
|
|
1522
|
+
});
|
|
1523
|
+
child.on("close", (code) => {
|
|
1524
|
+
if (timedOut || cancelled || cancellationError || args.terminateProcessGroupOnExit) {
|
|
1525
|
+
killProcessGroup("SIGKILL");
|
|
1526
|
+
}
|
|
1527
|
+
clearProcessTimers();
|
|
1528
|
+
if (timedOut) {
|
|
1529
|
+
reject(new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`));
|
|
1530
|
+
return;
|
|
1531
|
+
}
|
|
1532
|
+
if (cancellationError) {
|
|
1533
|
+
reject(cancellationError);
|
|
1534
|
+
return;
|
|
1535
|
+
}
|
|
1536
|
+
if (cancelled) {
|
|
1537
|
+
reject(new ProcessCancelledError(`${args.stage} was cancelled.`));
|
|
1538
|
+
return;
|
|
1539
|
+
}
|
|
1540
|
+
resolvePromise(code ?? 1);
|
|
1541
|
+
});
|
|
1542
|
+
});
|
|
1543
|
+
return { exitCode, stderr };
|
|
1544
|
+
} finally {
|
|
1545
|
+
if (logStream) {
|
|
1546
|
+
await new Promise((resolveEnd) => {
|
|
1547
|
+
logStream.end(resolveEnd);
|
|
1548
|
+
});
|
|
1217
1549
|
}
|
|
1218
|
-
});
|
|
1219
|
-
await mkdir2(dirname2(args.outputPath), { recursive: true });
|
|
1220
|
-
await writeFile2(args.outputPath, `${JSON.stringify(spec, null, 2)}
|
|
1221
|
-
`, "utf8");
|
|
1222
|
-
return spec;
|
|
1223
|
-
}
|
|
1224
|
-
async function initLocalRunnerConfigFile(args) {
|
|
1225
|
-
if (!args.force && await exists(args.outputPath)) {
|
|
1226
|
-
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
1227
1550
|
}
|
|
1228
|
-
const config = {
|
|
1229
|
-
artifactRoot: ".tt-local/artifacts",
|
|
1230
|
-
storeRoot: ".tt-local/store",
|
|
1231
|
-
evaluation: {
|
|
1232
|
-
inference: {
|
|
1233
|
-
device: "cuda"
|
|
1234
|
-
},
|
|
1235
|
-
scoring: {
|
|
1236
|
-
mode: "exact_match"
|
|
1237
|
-
},
|
|
1238
|
-
timeoutMs: 18e5
|
|
1239
|
-
}
|
|
1240
|
-
};
|
|
1241
|
-
await mkdir2(dirname2(args.outputPath), { recursive: true });
|
|
1242
|
-
await writeFile2(args.outputPath, `${JSON.stringify(config, null, 2)}
|
|
1243
|
-
`, "utf8");
|
|
1244
|
-
return config;
|
|
1245
1551
|
}
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1552
|
+
|
|
1553
|
+
// src/local-runtime/host-inventory.ts
|
|
1554
|
+
async function runCommand(command, args, options = {}) {
|
|
1555
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
1556
|
+
let interrupted;
|
|
1557
|
+
const result = await new Promise((resolveResult) => {
|
|
1558
|
+
let settled = false;
|
|
1559
|
+
let stdout = "";
|
|
1560
|
+
let stderr = "";
|
|
1561
|
+
let stopError;
|
|
1562
|
+
let forceKillTimer = null;
|
|
1563
|
+
const child = spawn2(command, args, {
|
|
1564
|
+
cwd: options.cwd,
|
|
1565
|
+
env: options.env ?? process.env,
|
|
1566
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1567
|
+
detached: process.platform !== "win32"
|
|
1568
|
+
});
|
|
1569
|
+
const killProcessGroup = (signal) => {
|
|
1570
|
+
if (child.pid && process.platform !== "win32") {
|
|
1571
|
+
try {
|
|
1572
|
+
process.kill(-child.pid, signal);
|
|
1573
|
+
return;
|
|
1574
|
+
} catch {
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
child.kill(signal);
|
|
1578
|
+
};
|
|
1579
|
+
const requestStop = (signal = "SIGTERM") => {
|
|
1580
|
+
killProcessGroup(signal);
|
|
1581
|
+
if (!forceKillTimer) {
|
|
1582
|
+
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), 5e3);
|
|
1583
|
+
forceKillTimer.unref();
|
|
1584
|
+
}
|
|
1585
|
+
};
|
|
1586
|
+
const onSigint = () => {
|
|
1587
|
+
interrupted = "interrupted by SIGINT";
|
|
1588
|
+
stopError = interrupted;
|
|
1589
|
+
requestStop("SIGINT");
|
|
1590
|
+
};
|
|
1591
|
+
const onSigterm = () => {
|
|
1592
|
+
interrupted = "interrupted by SIGTERM";
|
|
1593
|
+
stopError = interrupted;
|
|
1594
|
+
requestStop("SIGTERM");
|
|
1595
|
+
};
|
|
1596
|
+
const finish = (result2) => {
|
|
1597
|
+
if (settled) return;
|
|
1598
|
+
settled = true;
|
|
1599
|
+
clearTimeout(timer);
|
|
1600
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
1601
|
+
process.off("SIGINT", onSigint);
|
|
1602
|
+
process.off("SIGTERM", onSigterm);
|
|
1603
|
+
resolveResult(result2);
|
|
1604
|
+
};
|
|
1605
|
+
const timer = setTimeout(() => {
|
|
1606
|
+
stopError = `timed out after ${timeoutMs}ms`;
|
|
1607
|
+
requestStop();
|
|
1608
|
+
}, timeoutMs);
|
|
1609
|
+
timer.unref();
|
|
1610
|
+
process.once("SIGINT", onSigint);
|
|
1611
|
+
process.once("SIGTERM", onSigterm);
|
|
1612
|
+
child.stdout.setEncoding("utf8");
|
|
1613
|
+
child.stderr.setEncoding("utf8");
|
|
1614
|
+
child.stdout.on("data", (chunk) => {
|
|
1615
|
+
stdout += chunk;
|
|
1616
|
+
});
|
|
1617
|
+
child.stderr.on("data", (chunk) => {
|
|
1618
|
+
stderr += chunk;
|
|
1619
|
+
});
|
|
1620
|
+
child.on("error", (error) => {
|
|
1621
|
+
finish({ code: null, stdout, stderr, error: error.message });
|
|
1622
|
+
});
|
|
1623
|
+
child.on("close", (code) => {
|
|
1624
|
+
finish({ code: stopError ? null : code, stdout, stderr, error: stopError });
|
|
1625
|
+
});
|
|
1255
1626
|
});
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
function generatedPlaceholderIssues(request) {
|
|
1259
|
-
return placeholderIssues(request.spec_snapshot.system_prompt, request.spec_snapshot.examples);
|
|
1260
|
-
}
|
|
1261
|
-
function assertLocalRunInputReady(request) {
|
|
1262
|
-
const issues = generatedPlaceholderIssues(request);
|
|
1263
|
-
if (issues.length > 0) {
|
|
1264
|
-
throw new Error(`Edit the generated behavior spec before training: ${issues.join("; ")}.`);
|
|
1265
|
-
}
|
|
1266
|
-
}
|
|
1267
|
-
function foundationPlaceholderIssues(spec) {
|
|
1268
|
-
return placeholderIssues(spec.system_prompt, spec.examples);
|
|
1627
|
+
if (interrupted) throw new Error(interrupted);
|
|
1628
|
+
return result;
|
|
1269
1629
|
}
|
|
1270
|
-
function
|
|
1271
|
-
|
|
1272
|
-
if (issues.length > 0) {
|
|
1273
|
-
throw new Error(`Edit the generated foundation spec before training: ${issues.join("; ")}.`);
|
|
1274
|
-
}
|
|
1630
|
+
function firstLine(value) {
|
|
1631
|
+
return value.trim().split(/\r?\n/)[0] ?? "";
|
|
1275
1632
|
}
|
|
1276
|
-
function
|
|
1277
|
-
if (
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
}
|
|
1282
|
-
const {
|
|
1283
|
-
id,
|
|
1284
|
-
engine: _engine,
|
|
1285
|
-
hyperparameters,
|
|
1286
|
-
dataset_prebuilt: datasetPrebuilt,
|
|
1287
|
-
...specSnapshot
|
|
1288
|
-
} = spec;
|
|
1289
|
-
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
1290
|
-
return fineTuneRunRequestSchema.parse({
|
|
1291
|
-
run_id: options.runId ?? randomUUID2(),
|
|
1292
|
-
user_id: "local-user",
|
|
1293
|
-
behavior_spec_id: behaviorSpecId,
|
|
1294
|
-
run_number: 1,
|
|
1295
|
-
spec_snapshot: specSnapshot,
|
|
1296
|
-
hyperparameters,
|
|
1297
|
-
dataset_prebuilt: datasetPrebuilt
|
|
1298
|
-
});
|
|
1633
|
+
function parseMiB(value) {
|
|
1634
|
+
if (!value) return void 0;
|
|
1635
|
+
const parsed = Number(value.replace(/[^\d.]/g, ""));
|
|
1636
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return void 0;
|
|
1637
|
+
return Math.round(parsed * 1024 * 1024);
|
|
1299
1638
|
}
|
|
1300
|
-
function
|
|
1301
|
-
|
|
1302
|
-
const digest = createHash2("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
|
|
1303
|
-
digest[6] = digest[6] & 15 | 80;
|
|
1304
|
-
digest[8] = digest[8] & 63 | 128;
|
|
1305
|
-
const hex = digest.toString("hex");
|
|
1306
|
-
return [
|
|
1307
|
-
hex.slice(0, 8),
|
|
1308
|
-
hex.slice(8, 12),
|
|
1309
|
-
hex.slice(12, 16),
|
|
1310
|
-
hex.slice(16, 20),
|
|
1311
|
-
hex.slice(20)
|
|
1312
|
-
].join("-");
|
|
1639
|
+
function isNvidiaSmiDecorativeLine(line) {
|
|
1640
|
+
return /^(?:mon|tue|wed|thu|fri|sat|sun)\b/i.test(line) || line.startsWith("+---") || line.startsWith("|");
|
|
1313
1641
|
}
|
|
1314
|
-
|
|
1315
|
-
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile2(path, "utf8")), path);
|
|
1316
|
-
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
1317
|
-
if (request.success) {
|
|
1318
|
-
return { kind: "request", path, request: request.data };
|
|
1319
|
-
}
|
|
1320
|
-
const spec = localBehaviorSpecFileSchema.parse(raw);
|
|
1321
|
-
if (isFoundationSpecFile(spec)) {
|
|
1322
|
-
return { kind: "foundation-spec", path, spec };
|
|
1323
|
-
}
|
|
1642
|
+
function gpuFromName(name, index, extras = {}) {
|
|
1324
1643
|
return {
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1644
|
+
index,
|
|
1645
|
+
name,
|
|
1646
|
+
unified_memory: /spark|gb10/i.test(name),
|
|
1647
|
+
...extras
|
|
1329
1648
|
};
|
|
1330
1649
|
}
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
}
|
|
1346
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1347
|
-
throw new Error(`Invalid chat JSONL row ${index + 1}: expected an object`);
|
|
1348
|
-
}
|
|
1349
|
-
const messages = value.messages;
|
|
1350
|
-
if (!Array.isArray(messages) || ![2, 3].includes(messages.length) || messages.some(
|
|
1351
|
-
(message) => !message || typeof message !== "object" || Array.isArray(message) || !["system", "user", "assistant"].includes(String(message.role)) || typeof message.content !== "string"
|
|
1352
|
-
)) {
|
|
1353
|
-
throw new Error(
|
|
1354
|
-
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one user message, and one assistant answer`
|
|
1355
|
-
);
|
|
1650
|
+
function parseNvidiaGpus(stdout) {
|
|
1651
|
+
const gpus = [];
|
|
1652
|
+
for (const line of stdout.split(/\r?\n/).map((item) => item.trim()).filter(Boolean)) {
|
|
1653
|
+
if (isNvidiaSmiDecorativeLine(line)) continue;
|
|
1654
|
+
const parts = line.split(",").map((part) => part.trim());
|
|
1655
|
+
if (parts[0] && /^\d+$/.test(parts[0]) && parts.length >= 2) {
|
|
1656
|
+
const name = parts[1] || "NVIDIA GPU";
|
|
1657
|
+
gpus.push(gpuFromName(name, Number(parts[0]), {
|
|
1658
|
+
driver_version: parts[2] || void 0,
|
|
1659
|
+
memory_total_bytes: parseMiB(parts[3]),
|
|
1660
|
+
memory_free_bytes: parseMiB(parts[4]),
|
|
1661
|
+
compute_cap: parts[5] || void 0
|
|
1662
|
+
}));
|
|
1663
|
+
continue;
|
|
1356
1664
|
}
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1665
|
+
if (parts.length >= 2 && parts[0] && !/^(?:NVIDIA-SMI|Fail|Error)/i.test(parts[0])) {
|
|
1666
|
+
const name = parts[0];
|
|
1667
|
+
gpus.push(gpuFromName(name, gpus.length, {
|
|
1668
|
+
driver_version: parts[1] || void 0,
|
|
1669
|
+
memory_total_bytes: parseMiB(parts[2]),
|
|
1670
|
+
memory_free_bytes: parseMiB(parts[3])
|
|
1671
|
+
}));
|
|
1363
1672
|
}
|
|
1364
|
-
rows.push(row);
|
|
1365
|
-
jsonLines.push(JSON.stringify(row));
|
|
1366
1673
|
}
|
|
1367
|
-
|
|
1368
|
-
return {
|
|
1369
|
-
rows,
|
|
1370
|
-
jsonl: jsonLines.join("\n")
|
|
1371
|
-
};
|
|
1372
|
-
}
|
|
1373
|
-
async function normalizeChatJsonlForRelocation(path) {
|
|
1374
|
-
return (await loadNormalizedChatJsonl(path)).jsonl;
|
|
1674
|
+
return gpus;
|
|
1375
1675
|
}
|
|
1376
|
-
function
|
|
1377
|
-
const
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
parts.push(`Constraints:
|
|
1385
|
-
${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
|
|
1676
|
+
function parseNvidiaTableGpus(stdout) {
|
|
1677
|
+
const gpus = [];
|
|
1678
|
+
for (const line of stdout.split(/\r?\n/)) {
|
|
1679
|
+
const match = line.match(/^\|\s+(\d+)\s+(.+?)\s+(?:On|Off|N\/A)\s+\|/);
|
|
1680
|
+
if (!match) continue;
|
|
1681
|
+
const name = (match[2] ?? "").trim();
|
|
1682
|
+
if (!name || /^NVIDIA-SMI/i.test(name)) continue;
|
|
1683
|
+
gpus.push(gpuFromName(name, Number(match[1])));
|
|
1386
1684
|
}
|
|
1387
|
-
return
|
|
1388
|
-
}
|
|
1389
|
-
function exampleToChatRow(spec, example) {
|
|
1390
|
-
const system = buildSystemMessage(spec);
|
|
1391
|
-
return {
|
|
1392
|
-
messages: [
|
|
1393
|
-
{ role: "system", content: system },
|
|
1394
|
-
{ role: "user", content: example.input },
|
|
1395
|
-
{ role: "assistant", content: example.output }
|
|
1396
|
-
]
|
|
1397
|
-
};
|
|
1398
|
-
}
|
|
1399
|
-
function compileSpecToJsonl(spec) {
|
|
1400
|
-
return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
|
|
1401
|
-
}
|
|
1402
|
-
function examplesFromSpec(spec) {
|
|
1403
|
-
return spec.examples;
|
|
1685
|
+
return gpus;
|
|
1404
1686
|
}
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1687
|
+
var NVIDIA_CSV_QUERIES = [
|
|
1688
|
+
["--query-gpu=index,name,driver_version,memory.total,memory.free", "--format=csv,noheader,nounits"],
|
|
1689
|
+
["--query-gpu=name,driver_version,memory.total,memory.free", "--format=csv,noheader,nounits"],
|
|
1690
|
+
["--query-gpu=name,driver_version", "--format=csv,noheader"]
|
|
1691
|
+
];
|
|
1692
|
+
async function queryNvidiaGpus(env) {
|
|
1693
|
+
let last;
|
|
1694
|
+
for (const args of NVIDIA_CSV_QUERIES) {
|
|
1695
|
+
const result = await runCommand("nvidia-smi", args, { env, timeoutMs: 1e4 });
|
|
1696
|
+
last = result;
|
|
1697
|
+
if (result.code !== 0) continue;
|
|
1698
|
+
const gpus2 = parseNvidiaGpus(result.stdout);
|
|
1699
|
+
if (gpus2.length > 0) return { result, gpus: gpus2 };
|
|
1700
|
+
}
|
|
1701
|
+
const table = await runCommand("nvidia-smi", [], { env, timeoutMs: 1e4 });
|
|
1702
|
+
const gpus = table.code === 0 ? parseNvidiaTableGpus(table.stdout) : [];
|
|
1703
|
+
return { result: table.code === 0 || !last ? table : last, gpus };
|
|
1704
|
+
}
|
|
1705
|
+
async function diskCheck(name, path) {
|
|
1706
|
+
const resolvedPath = resolve7(path);
|
|
1707
|
+
try {
|
|
1708
|
+
await access(resolvedPath, constants.R_OK);
|
|
1709
|
+
const fs = await statfs(resolvedPath);
|
|
1710
|
+
const freeBytes = Number(fs.bavail) * Number(fs.bsize);
|
|
1711
|
+
return {
|
|
1712
|
+
name,
|
|
1713
|
+
path: resolvedPath,
|
|
1714
|
+
free_bytes: freeBytes,
|
|
1715
|
+
ok: true,
|
|
1716
|
+
message: `${resolvedPath} (${Math.round(freeBytes / 1024 ** 3 * 10) / 10} GiB available)`
|
|
1717
|
+
};
|
|
1718
|
+
} catch (error) {
|
|
1719
|
+
let probe = resolvedPath;
|
|
1720
|
+
for (let i = 0; i < 6; i += 1) {
|
|
1721
|
+
const parent = dirname5(probe);
|
|
1722
|
+
if (parent === probe) break;
|
|
1723
|
+
probe = parent;
|
|
1724
|
+
try {
|
|
1725
|
+
const fs = await statfs(probe);
|
|
1726
|
+
const freeBytes = Number(fs.bavail) * Number(fs.bsize);
|
|
1727
|
+
return {
|
|
1728
|
+
name,
|
|
1729
|
+
path: resolvedPath,
|
|
1730
|
+
free_bytes: freeBytes,
|
|
1731
|
+
ok: true,
|
|
1732
|
+
message: `${resolvedPath} not created yet; ${probe} has ${Math.round(freeBytes / 1024 ** 3 * 10) / 10} GiB available`
|
|
1733
|
+
};
|
|
1734
|
+
} catch {
|
|
1735
|
+
continue;
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
return {
|
|
1739
|
+
name,
|
|
1740
|
+
path: resolvedPath,
|
|
1741
|
+
ok: false,
|
|
1742
|
+
message: `${resolvedPath}: ${error instanceof Error ? error.message : String(error)}`
|
|
1743
|
+
};
|
|
1415
1744
|
}
|
|
1416
|
-
return examples;
|
|
1417
1745
|
}
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1746
|
+
function pythonProbeSource() {
|
|
1747
|
+
return [
|
|
1748
|
+
"import json",
|
|
1749
|
+
"import torch, transformers, peft, huggingface_hub",
|
|
1750
|
+
"payload = {'python_ok': True, 'torch': torch.__version__, 'transformers': transformers.__version__, 'peft': getattr(peft, '__version__', None), 'cuda_available': bool(torch.cuda.is_available())}",
|
|
1751
|
+
"cuda = torch.cuda.is_available()",
|
|
1752
|
+
"payload.update({'cuda_device': torch.cuda.get_device_name(0), 'compute_capability': list(torch.cuda.get_device_capability(0)), 'total_memory_bytes': torch.cuda.get_device_properties(0).total_memory, 'bf16_supported': bool(torch.cuda.is_bf16_supported())} if cuda else {})",
|
|
1753
|
+
"print(json.dumps(payload))"
|
|
1754
|
+
].join("; ");
|
|
1755
|
+
}
|
|
1756
|
+
function parsePythonProbeJson(text) {
|
|
1757
|
+
const line = [...text.split(/\r?\n/)].reverse().find((item) => item.trim().startsWith("{"));
|
|
1758
|
+
if (!line) return void 0;
|
|
1759
|
+
try {
|
|
1760
|
+
const parsed = JSON.parse(line);
|
|
1761
|
+
if (parsed.python_ok !== true) return void 0;
|
|
1762
|
+
return {
|
|
1763
|
+
ok: true,
|
|
1764
|
+
message: line.trim(),
|
|
1765
|
+
torch: typeof parsed.torch === "string" ? parsed.torch : void 0,
|
|
1766
|
+
transformers: typeof parsed.transformers === "string" ? parsed.transformers : void 0,
|
|
1767
|
+
peft: typeof parsed.peft === "string" ? parsed.peft : void 0,
|
|
1768
|
+
cuda_available: parsed.cuda_available === true,
|
|
1769
|
+
cuda_device: typeof parsed.cuda_device === "string" ? parsed.cuda_device : void 0,
|
|
1770
|
+
compute_capability: Array.isArray(parsed.compute_capability) ? parsed.compute_capability.filter((value) => typeof value === "number") : void 0,
|
|
1771
|
+
total_memory_bytes: typeof parsed.total_memory_bytes === "number" ? parsed.total_memory_bytes : void 0,
|
|
1772
|
+
bf16_supported: parsed.bf16_supported === true
|
|
1773
|
+
};
|
|
1774
|
+
} catch {
|
|
1775
|
+
return void 0;
|
|
1430
1776
|
}
|
|
1431
|
-
|
|
1432
|
-
|
|
1777
|
+
}
|
|
1778
|
+
async function collectPythonRuntime(config, env) {
|
|
1779
|
+
const entrypoint = buildBundledPythonCommand("-c", [pythonProbeSource()]);
|
|
1780
|
+
const childEnv = withBundledPythonEnvironment(
|
|
1781
|
+
withHuggingFaceCacheEnvironment(
|
|
1782
|
+
minimalMachineLearningEnvironment(env),
|
|
1783
|
+
config?.paths.modelCache
|
|
1784
|
+
)
|
|
1433
1785
|
);
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
if (
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
})),
|
|
1448
|
-
system: configuredSystem ?? embeddedSystem ?? "You are a helpful assistant."
|
|
1786
|
+
const result = await runCommand(entrypoint.command, entrypoint.commandArgs, {
|
|
1787
|
+
env: childEnv,
|
|
1788
|
+
timeoutMs: 18e5
|
|
1789
|
+
});
|
|
1790
|
+
if (result.code === 0) {
|
|
1791
|
+
return parsePythonProbeJson(result.stdout) ?? {
|
|
1792
|
+
ok: true,
|
|
1793
|
+
message: firstLine(result.stdout) || "Python runtime probe succeeded"
|
|
1794
|
+
};
|
|
1795
|
+
}
|
|
1796
|
+
return {
|
|
1797
|
+
ok: false,
|
|
1798
|
+
message: result.error ?? (firstLine(result.stderr) || `uv exited ${result.code}`)
|
|
1449
1799
|
};
|
|
1450
1800
|
}
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1801
|
+
async function collectHostInventory(options = {}) {
|
|
1802
|
+
const env = options.env ?? process.env;
|
|
1803
|
+
const config = options.config;
|
|
1804
|
+
const cwd = options.cwd ?? process.cwd();
|
|
1805
|
+
const nodeVersion = process.versions.node;
|
|
1806
|
+
const nodeMajor = Number(nodeVersion.split(".")[0]);
|
|
1807
|
+
const artifactRoot = resolve7(cwd, config?.artifactRoot ?? join5(cwd, DEFAULT_ARTIFACT_ROOT));
|
|
1808
|
+
const storeRoot = resolve7(config?.storeRoot ?? defaultStoreRoot(env));
|
|
1809
|
+
const modelCache = resolve7(
|
|
1810
|
+
config?.paths.modelCache ?? env.HF_HOME ?? join5(homedir3(), ".cache", "huggingface")
|
|
1811
|
+
);
|
|
1812
|
+
const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
|
|
1813
|
+
diskCheck("artifact-root", artifactRoot),
|
|
1814
|
+
diskCheck("store-root", storeRoot),
|
|
1815
|
+
diskCheck("model-cache", modelCache),
|
|
1816
|
+
runCommand("uv", ["--version"], { env, timeoutMs: 1e4 }),
|
|
1817
|
+
queryNvidiaGpus(env)
|
|
1818
|
+
]);
|
|
1819
|
+
const nvidia = nvidiaQueried.result;
|
|
1820
|
+
const gpus = nvidiaQueried.gpus;
|
|
1821
|
+
const inventory = {
|
|
1822
|
+
collected_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1823
|
+
quick: options.quick === true,
|
|
1824
|
+
node: {
|
|
1825
|
+
version: nodeVersion,
|
|
1826
|
+
major: nodeMajor,
|
|
1827
|
+
ok: nodeMajor >= 22
|
|
1828
|
+
},
|
|
1829
|
+
os: {
|
|
1830
|
+
platform: process.platform,
|
|
1831
|
+
arch: process.arch,
|
|
1832
|
+
type: type(),
|
|
1833
|
+
cpu_count: cpus().length,
|
|
1834
|
+
total_memory_bytes: totalmem(),
|
|
1835
|
+
free_memory_bytes: freemem()
|
|
1836
|
+
},
|
|
1837
|
+
uv: {
|
|
1838
|
+
ok: uvVersion.code === 0,
|
|
1839
|
+
version: uvVersion.code === 0 ? firstLine(uvVersion.stdout) : void 0,
|
|
1840
|
+
message: uvVersion.code === 0 ? firstLine(uvVersion.stdout) : uvVersion.error ?? (firstLine(uvVersion.stderr) || "uv is not available")
|
|
1841
|
+
},
|
|
1842
|
+
gpus,
|
|
1843
|
+
nvidia_smi: {
|
|
1844
|
+
ok: nvidia.code === 0,
|
|
1845
|
+
message: nvidia.code === 0 ? gpus[0] ? `${gpus[0].name}${gpus[0].memory_total_bytes ? ` (${Math.round(gpus[0].memory_total_bytes / 1024 ** 3)} GiB)` : ""}` : firstLine(nvidia.stdout) || "nvidia-smi reported a GPU" : nvidia.error ?? (firstLine(nvidia.stderr) || "nvidia-smi not available")
|
|
1846
|
+
},
|
|
1847
|
+
disks: [artifactDisk, storeDisk, cacheDisk]
|
|
1848
|
+
};
|
|
1849
|
+
if (options.quick !== true) {
|
|
1850
|
+
inventory.python = uvVersion.code === 0 ? await collectPythonRuntime(config, env) : { ok: false, message: "Python probe skipped because uv is not available" };
|
|
1457
1851
|
}
|
|
1852
|
+
return inventory;
|
|
1458
1853
|
}
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
}
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1854
|
+
|
|
1855
|
+
// src/local-runtime/hardware-snapshot.ts
|
|
1856
|
+
import { chmod, mkdir as mkdir3, readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
|
|
1857
|
+
import { dirname as dirname6, join as join6 } from "path";
|
|
1858
|
+
var HARDWARE_SNAPSHOT_VERSION = 1;
|
|
1859
|
+
var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
1860
|
+
function hardwareSnapshotPath(env = process.env) {
|
|
1861
|
+
return join6(getTunedTensorHome(env), "hardware.json");
|
|
1862
|
+
}
|
|
1863
|
+
async function readHardwareSnapshot(env = process.env) {
|
|
1864
|
+
const path = hardwareSnapshotPath(env);
|
|
1865
|
+
try {
|
|
1866
|
+
const parsed = JSON.parse(await readFile2(path, "utf8"));
|
|
1867
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return void 0;
|
|
1868
|
+
const snapshot = parsed;
|
|
1869
|
+
if (snapshot.version !== HARDWARE_SNAPSHOT_VERSION) return void 0;
|
|
1870
|
+
if (typeof snapshot.collected_at !== "string" || !snapshot.capabilities || !snapshot.inventory) {
|
|
1871
|
+
return void 0;
|
|
1471
1872
|
}
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
})
|
|
1873
|
+
const age = Date.now() - Date.parse(snapshot.collected_at);
|
|
1874
|
+
const stale = !Number.isFinite(age) || age > HARDWARE_SNAPSHOT_TTL_MS || age < 0;
|
|
1875
|
+
return { ...snapshot, stale, path };
|
|
1876
|
+
} catch (error) {
|
|
1877
|
+
if (error.code === "ENOENT") return void 0;
|
|
1878
|
+
return void 0;
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
async function writeHardwareSnapshot(snapshot, env = process.env) {
|
|
1882
|
+
const path = hardwareSnapshotPath(env);
|
|
1883
|
+
await mkdir3(dirname6(path), { recursive: true, mode: 448 });
|
|
1884
|
+
const gpu = snapshot.capabilities.gpu;
|
|
1885
|
+
const record = {
|
|
1886
|
+
version: HARDWARE_SNAPSHOT_VERSION,
|
|
1887
|
+
collected_at: snapshot.collected_at,
|
|
1888
|
+
quick: snapshot.quick,
|
|
1889
|
+
gpu_fingerprint: gpu ? { name: gpu.name, total_memory_bytes: gpu.memory_total_bytes } : void 0,
|
|
1890
|
+
inventory: snapshot.inventory,
|
|
1891
|
+
capabilities: snapshot.capabilities,
|
|
1892
|
+
summary: snapshot.summary ?? formatCapabilitySummary(snapshot.capabilities)
|
|
1893
|
+
};
|
|
1894
|
+
await writeFile2(path, `${JSON.stringify(record, null, 2)}
|
|
1895
|
+
`, { mode: 384 });
|
|
1896
|
+
await chmod(path, 384);
|
|
1897
|
+
return record;
|
|
1898
|
+
}
|
|
1899
|
+
function formatHostStatusLine(snapshot) {
|
|
1900
|
+
if (!snapshot) return "not inventoried (run tt hardware)";
|
|
1901
|
+
const stale = snapshot.stale ? `; stale \u2014 run tt hardware` : "";
|
|
1902
|
+
return `${snapshot.summary}${stale}`;
|
|
1903
|
+
}
|
|
1904
|
+
function formatAgentHostBlock(snapshot) {
|
|
1905
|
+
if (!snapshot) {
|
|
1906
|
+
return [
|
|
1907
|
+
"Host capability is not inventoried yet.",
|
|
1908
|
+
"When the user asks to examine this machine, GPU, VRAM, CUDA, or what can train, fine-tune, or infer here, call examine_hardware before recommending a base model, engine, or pipeline."
|
|
1909
|
+
].join(" ");
|
|
1910
|
+
}
|
|
1911
|
+
const stale = snapshot.stale ? " (stale \u2014 call examine_hardware to refresh)" : "";
|
|
1912
|
+
return [
|
|
1913
|
+
`Host capability (cached ${snapshot.collected_at}${stale}):`,
|
|
1914
|
+
snapshot.summary,
|
|
1915
|
+
"When the user wants to examine this host, GPU, or decide what can train/fine-tune/infer here, call examine_hardware before recommending a base model, engine, or pipeline.",
|
|
1916
|
+
"Recommend only workloads marked ready; mention tight as a caution. Never invent generic 7B/70B sizing."
|
|
1917
|
+
].join("\n");
|
|
1476
1918
|
}
|
|
1477
1919
|
|
|
1478
|
-
// src/local-runtime/
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1920
|
+
// src/local-runtime/hardware.ts
|
|
1921
|
+
async function assessHardware(options = {}) {
|
|
1922
|
+
const env = options.env ?? process.env;
|
|
1923
|
+
if (options.quick === true) {
|
|
1924
|
+
const existing = await readHardwareSnapshot(env);
|
|
1925
|
+
if (existing && !existing.quick && !existing.stale) {
|
|
1926
|
+
return {
|
|
1927
|
+
version: existing.version,
|
|
1928
|
+
collected_at: existing.collected_at,
|
|
1929
|
+
quick: existing.quick,
|
|
1930
|
+
gpu_fingerprint: existing.gpu_fingerprint,
|
|
1931
|
+
inventory: existing.inventory,
|
|
1932
|
+
capabilities: existing.capabilities,
|
|
1933
|
+
summary: existing.summary
|
|
1934
|
+
};
|
|
1935
|
+
}
|
|
1493
1936
|
}
|
|
1494
|
-
|
|
1937
|
+
const inventory = await collectHostInventory({
|
|
1938
|
+
config: options.config,
|
|
1939
|
+
quick: options.quick === true,
|
|
1940
|
+
env,
|
|
1941
|
+
cwd: options.cwd
|
|
1942
|
+
});
|
|
1943
|
+
const capabilities = evaluateCapabilities(inventory);
|
|
1944
|
+
return await writeHardwareSnapshot({
|
|
1945
|
+
collected_at: inventory.collected_at,
|
|
1946
|
+
quick: inventory.quick,
|
|
1947
|
+
inventory,
|
|
1948
|
+
capabilities,
|
|
1949
|
+
summary: formatCapabilitySummary(capabilities)
|
|
1950
|
+
}, env);
|
|
1495
1951
|
}
|
|
1496
1952
|
|
|
1497
1953
|
// src/local-runtime/store.ts
|
|
1498
|
-
import { appendFile, copyFile as copyFile2, mkdir as
|
|
1499
|
-
import {
|
|
1500
|
-
import {
|
|
1501
|
-
import { randomUUID as randomUUID3 } from "crypto";
|
|
1954
|
+
import { appendFile, chmod as chmod2, copyFile as copyFile2, lstat as lstat3, mkdir as mkdir4, readdir as readdir3, readFile as readFile3, rename as rename2, rm, stat as stat2, writeFile as writeFile3 } from "fs/promises";
|
|
1955
|
+
import { dirname as dirname7, join as join7, resolve as resolve8, sep as sep2 } from "path";
|
|
1956
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
1502
1957
|
function isTerminalRunState(state) {
|
|
1503
1958
|
return state.status === "stage_completed" || state.status === "completed" || state.status === "failed" || state.status === "cancelled" && state.current_stage === "cancelled" && Boolean(state.completed_at);
|
|
1504
1959
|
}
|
|
1505
1960
|
function defaultLocalHome() {
|
|
1506
|
-
return
|
|
1961
|
+
return defaultStoreRoot();
|
|
1507
1962
|
}
|
|
1508
1963
|
function localStorePaths(root) {
|
|
1509
1964
|
return {
|
|
1510
1965
|
root,
|
|
1511
|
-
specsDir:
|
|
1512
|
-
runsDir:
|
|
1513
|
-
modelsDir:
|
|
1514
|
-
datasetsDir:
|
|
1966
|
+
specsDir: join7(root, "specs"),
|
|
1967
|
+
runsDir: join7(root, "runs"),
|
|
1968
|
+
modelsDir: join7(root, "models"),
|
|
1969
|
+
datasetsDir: join7(root, "datasets")
|
|
1515
1970
|
};
|
|
1516
1971
|
}
|
|
1517
|
-
async function
|
|
1972
|
+
async function exists(path) {
|
|
1518
1973
|
try {
|
|
1519
|
-
await
|
|
1974
|
+
await stat2(path);
|
|
1520
1975
|
return true;
|
|
1521
1976
|
} catch {
|
|
1522
1977
|
return false;
|
|
1523
1978
|
}
|
|
1524
1979
|
}
|
|
1525
|
-
async function
|
|
1526
|
-
await
|
|
1527
|
-
const
|
|
1980
|
+
async function ensurePrivateDirectory(path) {
|
|
1981
|
+
await mkdir4(path, { recursive: true, mode: 448 });
|
|
1982
|
+
const metadata = await lstat3(path);
|
|
1983
|
+
if (metadata.isSymbolicLink()) {
|
|
1984
|
+
throw new Error(`Refusing symbolic link in local store: ${path}`);
|
|
1985
|
+
}
|
|
1986
|
+
if (!metadata.isDirectory()) {
|
|
1987
|
+
throw new Error(`Expected local store directory: ${path}`);
|
|
1988
|
+
}
|
|
1989
|
+
await chmod2(path, 448);
|
|
1990
|
+
}
|
|
1991
|
+
async function hardenPrivateTree(path) {
|
|
1992
|
+
const metadata = await lstat3(path);
|
|
1993
|
+
if (metadata.isSymbolicLink()) {
|
|
1994
|
+
throw new Error(`Refusing symbolic link in local store: ${path}`);
|
|
1995
|
+
}
|
|
1996
|
+
if (metadata.isDirectory()) {
|
|
1997
|
+
await chmod2(path, 448);
|
|
1998
|
+
for (const entry of await readdir3(path)) {
|
|
1999
|
+
await hardenPrivateTree(join7(path, entry));
|
|
2000
|
+
}
|
|
2001
|
+
return;
|
|
2002
|
+
}
|
|
2003
|
+
if (metadata.isFile()) {
|
|
2004
|
+
await chmod2(path, 384);
|
|
2005
|
+
return;
|
|
2006
|
+
}
|
|
2007
|
+
throw new Error(`Refusing unsupported filesystem entry in local store: ${path}`);
|
|
2008
|
+
}
|
|
2009
|
+
async function hardenExistingLocalStore(root) {
|
|
2010
|
+
const resolvedRoot = resolve8(root);
|
|
2011
|
+
let metadata;
|
|
2012
|
+
try {
|
|
2013
|
+
metadata = await lstat3(resolvedRoot);
|
|
2014
|
+
} catch (error) {
|
|
2015
|
+
if (error.code === "ENOENT") return false;
|
|
2016
|
+
throw error;
|
|
2017
|
+
}
|
|
2018
|
+
if (metadata.isSymbolicLink()) {
|
|
2019
|
+
throw new Error(`Refusing symbolic link in local store: ${resolvedRoot}`);
|
|
2020
|
+
}
|
|
2021
|
+
if (!metadata.isDirectory()) {
|
|
2022
|
+
throw new Error(`Expected local store directory: ${resolvedRoot}`);
|
|
2023
|
+
}
|
|
2024
|
+
await hardenPrivateTree(resolvedRoot);
|
|
2025
|
+
return true;
|
|
2026
|
+
}
|
|
2027
|
+
async function writePrivateJsonAtomic(path, value) {
|
|
2028
|
+
await ensurePrivateDirectory(dirname7(path));
|
|
2029
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.${randomUUID2()}.tmp`;
|
|
1528
2030
|
await writeFile3(tmp, `${JSON.stringify(value, null, 2)}
|
|
1529
|
-
`,
|
|
2031
|
+
`, {
|
|
2032
|
+
encoding: "utf8",
|
|
2033
|
+
mode: 384
|
|
2034
|
+
});
|
|
1530
2035
|
await rename2(tmp, path);
|
|
2036
|
+
await chmod2(path, 384);
|
|
1531
2037
|
}
|
|
1532
2038
|
async function readJson2(path) {
|
|
1533
|
-
return JSON.parse(await
|
|
2039
|
+
return JSON.parse(await readFile3(path, "utf8"));
|
|
1534
2040
|
}
|
|
1535
2041
|
async function appendJsonl(path, value) {
|
|
1536
|
-
await
|
|
2042
|
+
await ensurePrivateDirectory(dirname7(path));
|
|
1537
2043
|
await appendFile(path, `${JSON.stringify(value)}
|
|
1538
|
-
`,
|
|
2044
|
+
`, {
|
|
2045
|
+
encoding: "utf8",
|
|
2046
|
+
mode: 384
|
|
2047
|
+
});
|
|
2048
|
+
await chmod2(path, 384);
|
|
2049
|
+
}
|
|
2050
|
+
async function writePrivateText(path, value) {
|
|
2051
|
+
await ensurePrivateDirectory(dirname7(path));
|
|
2052
|
+
await writeFile3(path, value, { encoding: "utf8", mode: 384 });
|
|
2053
|
+
await chmod2(path, 384);
|
|
1539
2054
|
}
|
|
1540
2055
|
async function readJsonl(path) {
|
|
1541
|
-
if (!await
|
|
1542
|
-
return (await
|
|
2056
|
+
if (!await exists(path)) return [];
|
|
2057
|
+
return (await readFile3(path, "utf8")).split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
|
|
1543
2058
|
}
|
|
1544
2059
|
async function copyIfExists(from, to) {
|
|
1545
|
-
if (!await
|
|
1546
|
-
await
|
|
2060
|
+
if (!await exists(from)) return;
|
|
2061
|
+
await ensurePrivateDirectory(dirname7(to));
|
|
1547
2062
|
await copyFile2(from, to);
|
|
2063
|
+
await chmod2(to, 384);
|
|
1548
2064
|
}
|
|
1549
2065
|
async function childDirectoryNames(path) {
|
|
1550
2066
|
try {
|
|
@@ -1571,41 +2087,51 @@ function findByIdOrPrefix(records, id) {
|
|
|
1571
2087
|
return records.find((record) => record.id === id) ?? records.find((record) => record.id.startsWith(id));
|
|
1572
2088
|
}
|
|
1573
2089
|
function isPathInside(root, candidate) {
|
|
1574
|
-
const resolvedRoot =
|
|
1575
|
-
const resolvedCandidate =
|
|
2090
|
+
const resolvedRoot = resolve8(root);
|
|
2091
|
+
const resolvedCandidate = resolve8(candidate);
|
|
1576
2092
|
return resolvedCandidate === resolvedRoot || resolvedCandidate.startsWith(`${resolvedRoot}${sep2}`);
|
|
1577
2093
|
}
|
|
1578
2094
|
function createLocalStore(root = defaultLocalHome()) {
|
|
1579
|
-
const resolvedRoot =
|
|
2095
|
+
const resolvedRoot = resolve8(root);
|
|
1580
2096
|
const paths = localStorePaths(resolvedRoot);
|
|
1581
|
-
const runDir = (id) =>
|
|
1582
|
-
const runStatePath = (id) =>
|
|
1583
|
-
const runEventsPath = (id) =>
|
|
1584
|
-
const runRequestPath = (id) =>
|
|
1585
|
-
const runReportPath = (id) =>
|
|
1586
|
-
const cancellationPath = (id) =>
|
|
1587
|
-
const specDir = (id) =>
|
|
1588
|
-
const specPath = (id) =>
|
|
1589
|
-
const modelPath = (id) =>
|
|
1590
|
-
|
|
2097
|
+
const runDir = (id) => join7(paths.runsDir, id);
|
|
2098
|
+
const runStatePath = (id) => join7(runDir(id), "state.json");
|
|
2099
|
+
const runEventsPath = (id) => join7(runDir(id), "progress.jsonl");
|
|
2100
|
+
const runRequestPath = (id) => join7(runDir(id), "request.json");
|
|
2101
|
+
const runReportPath = (id) => join7(runDir(id), "run-report.json");
|
|
2102
|
+
const cancellationPath = (id) => join7(runDir(id), "cancel.requested");
|
|
2103
|
+
const specDir = (id) => join7(paths.specsDir, id);
|
|
2104
|
+
const specPath = (id) => join7(specDir(id), "spec.json");
|
|
2105
|
+
const modelPath = (id) => join7(paths.modelsDir, id, "model.json");
|
|
2106
|
+
let ensurePromise;
|
|
2107
|
+
async function ensureOnce() {
|
|
2108
|
+
await ensurePrivateDirectory(paths.root);
|
|
1591
2109
|
await Promise.all([
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
2110
|
+
ensurePrivateDirectory(paths.specsDir),
|
|
2111
|
+
ensurePrivateDirectory(paths.runsDir),
|
|
2112
|
+
ensurePrivateDirectory(paths.modelsDir),
|
|
2113
|
+
ensurePrivateDirectory(paths.datasetsDir)
|
|
1596
2114
|
]);
|
|
2115
|
+
await hardenExistingLocalStore(paths.root);
|
|
2116
|
+
}
|
|
2117
|
+
async function ensure() {
|
|
2118
|
+
ensurePromise ??= ensureOnce().catch((error) => {
|
|
2119
|
+
ensurePromise = void 0;
|
|
2120
|
+
throw error;
|
|
2121
|
+
});
|
|
2122
|
+
await ensurePromise;
|
|
1597
2123
|
}
|
|
1598
2124
|
async function writeRunState(state) {
|
|
1599
|
-
await
|
|
2125
|
+
await writePrivateJsonAtomic(runStatePath(state.id), state);
|
|
1600
2126
|
return state;
|
|
1601
2127
|
}
|
|
1602
2128
|
async function readSpecRecord(id) {
|
|
1603
2129
|
const path = specPath(id);
|
|
1604
|
-
if (!await
|
|
2130
|
+
if (!await exists(path)) return null;
|
|
1605
2131
|
const [spec, fileStats, directoryStats] = await Promise.all([
|
|
1606
2132
|
readJson2(path),
|
|
1607
|
-
|
|
1608
|
-
|
|
2133
|
+
stat2(path),
|
|
2134
|
+
stat2(specDir(id))
|
|
1609
2135
|
]);
|
|
1610
2136
|
const createdAt = directoryStats.birthtimeMs > 0 ? directoryStats.birthtime : directoryStats.ctime;
|
|
1611
2137
|
return {
|
|
@@ -1631,7 +2157,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1631
2157
|
const records = [];
|
|
1632
2158
|
for (const id of await childDirectoryNames(paths.runsDir)) {
|
|
1633
2159
|
const path = runStatePath(id);
|
|
1634
|
-
if (await
|
|
2160
|
+
if (await exists(path)) records.push(await readJson2(path));
|
|
1635
2161
|
}
|
|
1636
2162
|
return records.sort(compareUpdatedAt);
|
|
1637
2163
|
}
|
|
@@ -1640,27 +2166,27 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1640
2166
|
const records = [];
|
|
1641
2167
|
for (const id of await childDirectoryNames(paths.modelsDir)) {
|
|
1642
2168
|
const path = modelPath(id);
|
|
1643
|
-
if (await
|
|
2169
|
+
if (await exists(path)) records.push(await readJson2(path));
|
|
1644
2170
|
}
|
|
1645
2171
|
return records.sort(compareCreatedAt);
|
|
1646
2172
|
}
|
|
1647
2173
|
async function importSpecRecord(specId, spec) {
|
|
1648
2174
|
await ensure();
|
|
1649
|
-
await
|
|
1650
|
-
await
|
|
2175
|
+
await ensurePrivateDirectory(specDir(specId));
|
|
2176
|
+
await writePrivateJsonAtomic(specPath(specId), spec);
|
|
1651
2177
|
const record = await readSpecRecord(specId);
|
|
1652
2178
|
if (!record) throw new Error(`Failed to persist spec: ${specId}`);
|
|
1653
2179
|
return record;
|
|
1654
2180
|
}
|
|
1655
2181
|
async function appendRunEvent(state, event) {
|
|
1656
2182
|
const row = {
|
|
1657
|
-
id:
|
|
2183
|
+
id: randomUUID2(),
|
|
1658
2184
|
run_id: state.id,
|
|
1659
2185
|
occurred_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1660
2186
|
...event
|
|
1661
2187
|
};
|
|
1662
2188
|
await appendJsonl(runEventsPath(state.id), row);
|
|
1663
|
-
await appendJsonl(
|
|
2189
|
+
await appendJsonl(join7(state.artifact_dir, "progress.jsonl"), {
|
|
1664
2190
|
at: row.occurred_at,
|
|
1665
2191
|
stage: row.stage,
|
|
1666
2192
|
status: row.status,
|
|
@@ -1690,6 +2216,10 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1690
2216
|
async function preserveCancellationRequestState(runId) {
|
|
1691
2217
|
const previous = await getRun(runId);
|
|
1692
2218
|
if (isTerminalRunState(previous)) return previous;
|
|
2219
|
+
await Promise.all([
|
|
2220
|
+
rm(join7(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true }),
|
|
2221
|
+
rm(runReportPath(previous.id), { force: true })
|
|
2222
|
+
]);
|
|
1693
2223
|
if (previous.status === "cancelled" && previous.current_stage === "cancel_requested") return previous;
|
|
1694
2224
|
const requested = {
|
|
1695
2225
|
...previous,
|
|
@@ -1704,19 +2234,20 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1704
2234
|
}
|
|
1705
2235
|
async function resolveRunId(id) {
|
|
1706
2236
|
await ensure();
|
|
1707
|
-
if (await
|
|
2237
|
+
if (await exists(runStatePath(id))) return id;
|
|
1708
2238
|
const record = findByIdOrPrefix(await listRunRecords(), id);
|
|
1709
2239
|
if (!record) throw new Error(`Run not found: ${id}`);
|
|
1710
2240
|
return record.id;
|
|
1711
2241
|
}
|
|
1712
2242
|
async function getRun(id) {
|
|
2243
|
+
await ensure();
|
|
1713
2244
|
return readJson2(runStatePath(await resolveRunId(id)));
|
|
1714
2245
|
}
|
|
1715
2246
|
async function registerModelRecord(args) {
|
|
1716
2247
|
if (!args.training.model_artifact_uri || args.training.metrics?.dry_run === true) return null;
|
|
1717
2248
|
const previous = await getRun(args.request.run_id);
|
|
1718
2249
|
const modelId = `local-${args.request.run_id}`;
|
|
1719
|
-
const existingModel = await
|
|
2250
|
+
const existingModel = await exists(modelPath(modelId)) ? await readJson2(modelPath(modelId)) : void 0;
|
|
1720
2251
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1721
2252
|
const model = {
|
|
1722
2253
|
id: modelId,
|
|
@@ -1730,7 +2261,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1730
2261
|
metrics: args.training.metrics,
|
|
1731
2262
|
created_at: existingModel?.created_at ?? args.createdAt ?? now
|
|
1732
2263
|
};
|
|
1733
|
-
await
|
|
2264
|
+
await writePrivateJsonAtomic(modelPath(model.id), model);
|
|
1734
2265
|
const alreadyRegistered = previous.model_id === modelId;
|
|
1735
2266
|
const state = {
|
|
1736
2267
|
...previous,
|
|
@@ -1780,7 +2311,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1780
2311
|
updated_at: now,
|
|
1781
2312
|
started_at: now
|
|
1782
2313
|
};
|
|
1783
|
-
await
|
|
2314
|
+
await writePrivateJsonAtomic(runRequestPath(request.run_id), request);
|
|
1784
2315
|
await writeRunState(state);
|
|
1785
2316
|
await appendRunEvent(state, { stage: "queued", status: "queued", message: "Run queued." });
|
|
1786
2317
|
return state;
|
|
@@ -1793,7 +2324,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1793
2324
|
);
|
|
1794
2325
|
}
|
|
1795
2326
|
await importSpecRecord(request.behavior_spec_id, request.spec_snapshot);
|
|
1796
|
-
await
|
|
2327
|
+
await writePrivateJsonAtomic(runRequestPath(request.run_id), request);
|
|
1797
2328
|
const state = {
|
|
1798
2329
|
...previous,
|
|
1799
2330
|
artifact_dir: artifactDir ?? previous.artifact_dir,
|
|
@@ -1806,7 +2337,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1806
2337
|
},
|
|
1807
2338
|
async updateRun({ runId, status, stage, message, details }) {
|
|
1808
2339
|
const previous = await getRun(runId);
|
|
1809
|
-
if (status !== "cancelled" && await
|
|
2340
|
+
if (status !== "cancelled" && await exists(cancellationPath(previous.id))) {
|
|
1810
2341
|
return preserveCancellationRequestState(previous.id);
|
|
1811
2342
|
}
|
|
1812
2343
|
const active = status !== "stage_completed" && status !== "completed" && status !== "failed" && status !== "cancelled";
|
|
@@ -1821,7 +2352,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1821
2352
|
updated_at: now
|
|
1822
2353
|
};
|
|
1823
2354
|
await writeRunState(state);
|
|
1824
|
-
if (status !== "cancelled" && await
|
|
2355
|
+
if (status !== "cancelled" && await exists(cancellationPath(previous.id))) {
|
|
1825
2356
|
return preserveCancellationRequestState(previous.id);
|
|
1826
2357
|
}
|
|
1827
2358
|
await appendRunEvent(state, {
|
|
@@ -1833,11 +2364,19 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1833
2364
|
return state;
|
|
1834
2365
|
},
|
|
1835
2366
|
async completeRun(report, artifactDir, reportPath) {
|
|
1836
|
-
|
|
2367
|
+
await ensure();
|
|
2368
|
+
if (await exists(cancellationPath(report.run_id))) {
|
|
1837
2369
|
return preserveCancellationRequestState(report.run_id);
|
|
1838
2370
|
}
|
|
1839
2371
|
const request = await readJson2(runRequestPath(report.run_id));
|
|
1840
2372
|
const completedAt = report.created_at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
2373
|
+
if (!await exists(reportPath)) {
|
|
2374
|
+
throw new Error(`Run report not found: ${reportPath}`);
|
|
2375
|
+
}
|
|
2376
|
+
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
2377
|
+
if (await exists(cancellationPath(report.run_id))) {
|
|
2378
|
+
return preserveCancellationRequestState(report.run_id);
|
|
2379
|
+
}
|
|
1841
2380
|
const model = await registerModelRecord({
|
|
1842
2381
|
request,
|
|
1843
2382
|
training: report.training,
|
|
@@ -1845,7 +2384,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1845
2384
|
createdAt: completedAt
|
|
1846
2385
|
});
|
|
1847
2386
|
if (!model) {
|
|
1848
|
-
await rm(
|
|
2387
|
+
await rm(join7(paths.modelsDir, `local-${report.run_id}`), { recursive: true, force: true });
|
|
1849
2388
|
}
|
|
1850
2389
|
const previous = await getRun(report.run_id);
|
|
1851
2390
|
const state = {
|
|
@@ -1859,11 +2398,13 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1859
2398
|
completed_at: completedAt,
|
|
1860
2399
|
updated_at: completedAt
|
|
1861
2400
|
};
|
|
2401
|
+
if (await exists(cancellationPath(report.run_id))) {
|
|
2402
|
+
return preserveCancellationRequestState(report.run_id);
|
|
2403
|
+
}
|
|
1862
2404
|
await writeRunState(state);
|
|
1863
|
-
if (await
|
|
2405
|
+
if (await exists(cancellationPath(report.run_id))) {
|
|
1864
2406
|
return preserveCancellationRequestState(report.run_id);
|
|
1865
2407
|
}
|
|
1866
|
-
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
1867
2408
|
await appendRunEvent(state, {
|
|
1868
2409
|
stage: "completed",
|
|
1869
2410
|
status: "completed",
|
|
@@ -1877,7 +2418,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1877
2418
|
const previous = await getRun(runId);
|
|
1878
2419
|
if (options.report) await rm(runReportPath(previous.id), { force: true });
|
|
1879
2420
|
if (options.model) {
|
|
1880
|
-
await rm(
|
|
2421
|
+
await rm(join7(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true });
|
|
1881
2422
|
}
|
|
1882
2423
|
const state = {
|
|
1883
2424
|
...previous,
|
|
@@ -1892,7 +2433,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1892
2433
|
async failRun(runId, error) {
|
|
1893
2434
|
const previous = await getRun(runId);
|
|
1894
2435
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1895
|
-
if (await
|
|
2436
|
+
if (await exists(cancellationPath(previous.id))) {
|
|
1896
2437
|
return preserveCancellationRequestState(previous.id);
|
|
1897
2438
|
}
|
|
1898
2439
|
const state = {
|
|
@@ -1905,7 +2446,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1905
2446
|
updated_at: now
|
|
1906
2447
|
};
|
|
1907
2448
|
await writeRunState(state);
|
|
1908
|
-
if (await
|
|
2449
|
+
if (await exists(cancellationPath(previous.id))) {
|
|
1909
2450
|
return preserveCancellationRequestState(previous.id);
|
|
1910
2451
|
}
|
|
1911
2452
|
await appendRunEvent(state, { stage: "failed", status: "failed", message: error });
|
|
@@ -1916,8 +2457,8 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1916
2457
|
if (isTerminalRunState(state) || state.status === "cancelled" && state.current_stage === "cancel_requested") {
|
|
1917
2458
|
return;
|
|
1918
2459
|
}
|
|
1919
|
-
await
|
|
1920
|
-
|
|
2460
|
+
await writePrivateText(cancellationPath(state.id), `${(/* @__PURE__ */ new Date()).toISOString()}
|
|
2461
|
+
`);
|
|
1921
2462
|
const latest = await getRun(state.id);
|
|
1922
2463
|
if (isTerminalRunState(latest)) {
|
|
1923
2464
|
if (latest.status !== "cancelled") await rm(cancellationPath(state.id), { force: true });
|
|
@@ -1940,7 +2481,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1940
2481
|
},
|
|
1941
2482
|
async isCancellationRequested(runId) {
|
|
1942
2483
|
const state = await getRun(runId);
|
|
1943
|
-
return state.status === "cancelled" || await
|
|
2484
|
+
return state.status === "cancelled" || await exists(cancellationPath(state.id));
|
|
1944
2485
|
},
|
|
1945
2486
|
async listRuns() {
|
|
1946
2487
|
return listRunRecords();
|
|
@@ -1950,356 +2491,654 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1950
2491
|
const state = await getRun(id);
|
|
1951
2492
|
return readJsonl(runEventsPath(state.id));
|
|
1952
2493
|
},
|
|
1953
|
-
async getRunReport(id) {
|
|
1954
|
-
const state = await getRun(id);
|
|
1955
|
-
const candidates = [];
|
|
1956
|
-
if (state.report_path) {
|
|
1957
|
-
const reportPath =
|
|
1958
|
-
const allowedRoots = [resolvedRoot];
|
|
1959
|
-
if (state.artifact_dir) allowedRoots.push(
|
|
1960
|
-
if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) {
|
|
1961
|
-
candidates.push(reportPath);
|
|
2494
|
+
async getRunReport(id) {
|
|
2495
|
+
const state = await getRun(id);
|
|
2496
|
+
const candidates = [];
|
|
2497
|
+
if (state.report_path) {
|
|
2498
|
+
const reportPath = resolve8(state.report_path);
|
|
2499
|
+
const allowedRoots = [resolvedRoot];
|
|
2500
|
+
if (state.artifact_dir) allowedRoots.push(resolve8(state.artifact_dir));
|
|
2501
|
+
if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) {
|
|
2502
|
+
candidates.push(reportPath);
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
candidates.push(runReportPath(state.id));
|
|
2506
|
+
for (const path of candidates) {
|
|
2507
|
+
if (await exists(path)) return readJson2(path);
|
|
2508
|
+
}
|
|
2509
|
+
throw new Error(`Run has no report yet: ${id}`);
|
|
2510
|
+
},
|
|
2511
|
+
async listModels() {
|
|
2512
|
+
return listModelRecords();
|
|
2513
|
+
},
|
|
2514
|
+
async getModel(id) {
|
|
2515
|
+
const record = findByIdOrPrefix(await listModelRecords(), id);
|
|
2516
|
+
if (!record) throw new Error(`Model not found: ${id}`);
|
|
2517
|
+
return record;
|
|
2518
|
+
}
|
|
2519
|
+
};
|
|
2520
|
+
}
|
|
2521
|
+
|
|
2522
|
+
// src/local-runtime/contracts.ts
|
|
2523
|
+
import { z } from "zod";
|
|
2524
|
+
var datasetFormatSchema = z.literal("chat_jsonl");
|
|
2525
|
+
var baseModelRevisionSchema = z.string().regex(/^[0-9a-f]{40}$/i, "base_model_revision must be a 40-character Hugging Face commit SHA").transform((value) => value.toLowerCase());
|
|
2526
|
+
var behaviorSpecExampleSchema = z.object({
|
|
2527
|
+
input: z.string().min(1),
|
|
2528
|
+
output: z.string().min(1)
|
|
2529
|
+
}).strict();
|
|
2530
|
+
var specSnapshotSchema = z.object({
|
|
2531
|
+
name: z.string().min(1),
|
|
2532
|
+
description: z.string().default(""),
|
|
2533
|
+
system_prompt: z.string().default(""),
|
|
2534
|
+
guidelines: z.array(z.string()).default([]),
|
|
2535
|
+
examples: z.array(behaviorSpecExampleSchema).default([]),
|
|
2536
|
+
constraints: z.array(z.string()).default([]),
|
|
2537
|
+
base_model: z.string().transform((value) => canonicalizeTrainingModel(value))
|
|
2538
|
+
}).strict();
|
|
2539
|
+
var foundationHyperparametersSchema = z.object({
|
|
2540
|
+
vocab_size: z.number().int().min(64).max(65536).default(256),
|
|
2541
|
+
max_chars: z.number().int().min(1e3).max(2e9).default(2e4),
|
|
2542
|
+
depth: z.number().int().min(1).max(64).default(2),
|
|
2543
|
+
pretrain_steps: z.number().int().min(1).max(1e7).default(2),
|
|
2544
|
+
finetune_steps: z.number().int().min(1).max(1e7).default(2),
|
|
2545
|
+
rl_steps: z.number().int().min(0).max(1e7).default(0),
|
|
2546
|
+
batch_size: z.number().int().min(1).max(1024).default(2),
|
|
2547
|
+
sequence_length: z.number().int().min(16).max(8192).default(64),
|
|
2548
|
+
nproc_per_node: z.number().int().min(1).max(16).default(1)
|
|
2549
|
+
}).strict();
|
|
2550
|
+
var localFoundationSpecFileSchema = z.object({
|
|
2551
|
+
engine: z.literal("foundation"),
|
|
2552
|
+
id: z.string().uuid().optional(),
|
|
2553
|
+
name: z.string().min(1),
|
|
2554
|
+
description: z.string().default(""),
|
|
2555
|
+
system_prompt: z.string().default(""),
|
|
2556
|
+
guidelines: z.array(z.string()).default([]),
|
|
2557
|
+
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
2558
|
+
constraints: z.array(z.string()).default([]),
|
|
2559
|
+
foundation: foundationHyperparametersSchema
|
|
2560
|
+
}).strict();
|
|
2561
|
+
var datasetPrebuiltSchema = z.object({
|
|
2562
|
+
training: z.string().min(1),
|
|
2563
|
+
validation: z.string().min(1).optional(),
|
|
2564
|
+
test: z.string().min(1).optional(),
|
|
2565
|
+
format: datasetFormatSchema.default("chat_jsonl")
|
|
2566
|
+
}).strict().superRefine((dataset, context) => {
|
|
2567
|
+
if (!dataset.validation && !dataset.test) {
|
|
2568
|
+
context.addIssue({
|
|
2569
|
+
code: z.ZodIssueCode.custom,
|
|
2570
|
+
path: ["validation"],
|
|
2571
|
+
message: "Provide a distinct validation or test split for held-out evaluation"
|
|
2572
|
+
});
|
|
2573
|
+
}
|
|
2574
|
+
});
|
|
2575
|
+
var fineTuneHyperparametersSchema = z.object({
|
|
2576
|
+
n_epochs: z.number().int().min(1).max(20).default(1),
|
|
2577
|
+
learning_rate: z.number().positive().optional(),
|
|
2578
|
+
batch_size: z.number().int().min(1).optional(),
|
|
2579
|
+
lora_rank: z.number().int().min(1).optional(),
|
|
2580
|
+
lora_alpha: z.number().int().min(1).optional(),
|
|
2581
|
+
lora_dropout: z.number().min(0).max(1).optional(),
|
|
2582
|
+
max_seq_length: z.number().int().min(128).max(32768).optional(),
|
|
2583
|
+
gradient_accumulation_steps: z.number().int().min(1).optional(),
|
|
2584
|
+
max_eval_examples: z.number().int().min(1).optional(),
|
|
2585
|
+
/** Immutable Hugging Face commit used by every stage of the run. */
|
|
2586
|
+
base_model_revision: baseModelRevisionSchema.optional()
|
|
2587
|
+
}).strict();
|
|
2588
|
+
var fineTuneRunRequestSchema = z.object({
|
|
2589
|
+
run_id: z.string().uuid(),
|
|
2590
|
+
user_id: z.string().min(1),
|
|
2591
|
+
behavior_spec_id: z.string().uuid(),
|
|
2592
|
+
run_number: z.number().int().min(1),
|
|
2593
|
+
spec_snapshot: specSnapshotSchema,
|
|
2594
|
+
hyperparameters: fineTuneHyperparametersSchema.default({ n_epochs: 1 }),
|
|
2595
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
2596
|
+
}).strict().superRefine((request, context) => {
|
|
2597
|
+
if (request.spec_snapshot.examples.length === 0 && !request.dataset_prebuilt) {
|
|
2598
|
+
context.addIssue({
|
|
2599
|
+
code: z.ZodIssueCode.custom,
|
|
2600
|
+
path: ["spec_snapshot", "examples"],
|
|
2601
|
+
message: "Add examples or provide dataset_prebuilt training and validation files"
|
|
2602
|
+
});
|
|
2603
|
+
}
|
|
2604
|
+
});
|
|
2605
|
+
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
2606
|
+
engine: z.literal("adapter").optional(),
|
|
2607
|
+
id: z.string().uuid().optional(),
|
|
2608
|
+
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
2609
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
2610
|
+
}).strict();
|
|
2611
|
+
var localBehaviorSpecFileSchema = z.union([
|
|
2612
|
+
localFoundationSpecFileSchema,
|
|
2613
|
+
localAdapterSpecFileSchema
|
|
2614
|
+
]);
|
|
2615
|
+
var evalExampleResultSchema = z.object({
|
|
2616
|
+
prompt: z.string(),
|
|
2617
|
+
expected: z.string(),
|
|
2618
|
+
actual: z.string(),
|
|
2619
|
+
passed: z.boolean(),
|
|
2620
|
+
score: z.number().min(0).max(1),
|
|
2621
|
+
reasoning: z.string().nullable(),
|
|
2622
|
+
latency_ms: z.number().int().nonnegative(),
|
|
2623
|
+
scored_by: z.enum(["exact_match", "json_fields", "heuristic"]).optional()
|
|
2624
|
+
}).strict();
|
|
2625
|
+
var jsonFieldMetricsSchema = z.object({
|
|
2626
|
+
fields: z.array(z.string()),
|
|
2627
|
+
valid_json_count: z.number().int().nonnegative(),
|
|
2628
|
+
valid_json_rate: z.number().min(0).max(1),
|
|
2629
|
+
schema_match_count: z.number().int().nonnegative(),
|
|
2630
|
+
schema_match_rate: z.number().min(0).max(1),
|
|
2631
|
+
all_fields_match_count: z.number().int().nonnegative(),
|
|
2632
|
+
all_fields_match_rate: z.number().min(0).max(1),
|
|
2633
|
+
field_accuracy: z.record(z.string(), z.object({
|
|
2634
|
+
correct: z.number().int().nonnegative(),
|
|
2635
|
+
total: z.number().int().nonnegative(),
|
|
2636
|
+
accuracy: z.number().min(0).max(1)
|
|
2637
|
+
}).strict())
|
|
2638
|
+
}).strict();
|
|
2639
|
+
var evalSplitSchema = z.enum([
|
|
2640
|
+
"spec_examples",
|
|
2641
|
+
"spec_holdout",
|
|
2642
|
+
"prebuilt_test",
|
|
2643
|
+
"prebuilt_validation",
|
|
2644
|
+
"general_regression"
|
|
2645
|
+
]);
|
|
2646
|
+
var evalReportSchema = z.object({
|
|
2647
|
+
kind: z.enum(["baseline", "candidate"]),
|
|
2648
|
+
model_id: z.string(),
|
|
2649
|
+
total: z.number().int().nonnegative(),
|
|
2650
|
+
eval_examples_total: z.number().int().nonnegative(),
|
|
2651
|
+
eval_examples_used: z.number().int().nonnegative(),
|
|
2652
|
+
eval_truncated: z.boolean(),
|
|
2653
|
+
eval_split: evalSplitSchema.optional(),
|
|
2654
|
+
eval_sample_seed: z.number().int().nullable().optional(),
|
|
2655
|
+
avg_score: z.number().min(0).max(1),
|
|
2656
|
+
pass_rate: z.number().min(0).max(1),
|
|
2657
|
+
exact_match_rate: z.number().min(0).max(1),
|
|
2658
|
+
avg_token_f1: z.number().min(0).max(1).optional(),
|
|
2659
|
+
avg_latency_ms: z.number().int().nonnegative(),
|
|
2660
|
+
cached: z.boolean().optional(),
|
|
2661
|
+
cache_key: z.string().optional(),
|
|
2662
|
+
results: z.array(evalExampleResultSchema),
|
|
2663
|
+
artifact_uri: z.string(),
|
|
2664
|
+
scoring_method: z.enum(["heuristic", "exact_match", "json_fields"]),
|
|
2665
|
+
inference_provider: z.enum(["none", "transformers"]).optional(),
|
|
2666
|
+
scoring_mode: z.enum(["exact_match", "json_fields"]).optional(),
|
|
2667
|
+
json_field_metrics: jsonFieldMetricsSchema.optional(),
|
|
2668
|
+
generation_config: z.record(z.string(), z.unknown()).optional(),
|
|
2669
|
+
log_uri: z.string().optional()
|
|
2670
|
+
}).strict();
|
|
2671
|
+
var comparisonReportSchema = z.object({
|
|
2672
|
+
avg_score_delta: z.number(),
|
|
2673
|
+
pass_rate_delta: z.number(),
|
|
2674
|
+
exact_match_rate_delta: z.number(),
|
|
2675
|
+
token_f1_delta: z.number().optional(),
|
|
2676
|
+
regressions: z.number().int().nonnegative(),
|
|
2677
|
+
improvements: z.number().int().nonnegative(),
|
|
2678
|
+
regressed_examples: z.array(z.object({
|
|
2679
|
+
prompt: z.string(),
|
|
2680
|
+
old_score: z.number(),
|
|
2681
|
+
new_score: z.number()
|
|
2682
|
+
}).strict())
|
|
2683
|
+
}).strict();
|
|
2684
|
+
var trainingReportSchema = z.object({
|
|
2685
|
+
provider: z.literal("local-uv"),
|
|
2686
|
+
training_job_name: z.string(),
|
|
2687
|
+
model_artifact_uri: z.string().optional(),
|
|
2688
|
+
base_model_artifact_uri: z.string().optional(),
|
|
2689
|
+
metrics: z.record(z.string(), z.unknown()).nullable(),
|
|
2690
|
+
exit_code: z.number().int().nullable(),
|
|
2691
|
+
log_uri: z.string(),
|
|
2692
|
+
command: z.array(z.string()).optional()
|
|
2693
|
+
}).strict();
|
|
2694
|
+
var runReportSchema = z.object({
|
|
2695
|
+
run_id: z.string().uuid(),
|
|
2696
|
+
behavior_spec_id: z.string().uuid(),
|
|
2697
|
+
user_id: z.string(),
|
|
2698
|
+
run_number: z.number().int().min(1),
|
|
2699
|
+
base_model: z.string(),
|
|
2700
|
+
fine_tuned_model_id: z.string(),
|
|
2701
|
+
status: z.enum(["completed", "failed"]),
|
|
2702
|
+
baseline: evalReportSchema,
|
|
2703
|
+
candidate: evalReportSchema,
|
|
2704
|
+
comparison: comparisonReportSchema,
|
|
2705
|
+
general_regression: z.object({
|
|
2706
|
+
dataset_uri: z.string(),
|
|
2707
|
+
dataset_sha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
2708
|
+
baseline: evalReportSchema,
|
|
2709
|
+
candidate: evalReportSchema,
|
|
2710
|
+
comparison: comparisonReportSchema,
|
|
2711
|
+
policy: z.object({
|
|
2712
|
+
max_score_drop: z.number().min(0).max(1),
|
|
2713
|
+
max_pass_rate_drop: z.number().min(0).max(1)
|
|
2714
|
+
}).strict(),
|
|
2715
|
+
passed: z.boolean(),
|
|
2716
|
+
failures: z.array(z.string())
|
|
2717
|
+
}).strict().optional(),
|
|
2718
|
+
training: trainingReportSchema,
|
|
2719
|
+
artifact_uris: z.object({
|
|
2720
|
+
dataset: z.string(),
|
|
2721
|
+
baseline_eval: z.string(),
|
|
2722
|
+
candidate_eval: z.string(),
|
|
2723
|
+
general_baseline_eval: z.string().optional(),
|
|
2724
|
+
general_candidate_eval: z.string().optional(),
|
|
2725
|
+
report: z.string()
|
|
2726
|
+
}).strict(),
|
|
2727
|
+
run_metadata: z.object({
|
|
2728
|
+
base_model: z.string(),
|
|
2729
|
+
fine_tuned_model_id: z.string(),
|
|
2730
|
+
dataset_prebuilt: z.boolean(),
|
|
2731
|
+
dataset_format: datasetFormatSchema.nullable().optional(),
|
|
2732
|
+
dataset_uri: z.string(),
|
|
2733
|
+
spec_example_count: z.number().int().nonnegative(),
|
|
2734
|
+
training_example_count: z.number().int().nonnegative().nullable(),
|
|
2735
|
+
eval_examples_total: z.number().int().nonnegative(),
|
|
2736
|
+
eval_examples_used: z.number().int().nonnegative(),
|
|
2737
|
+
eval_split: evalSplitSchema.optional(),
|
|
2738
|
+
eval_sample_seed: z.number().int().nullable().optional(),
|
|
2739
|
+
started_at: z.string(),
|
|
2740
|
+
completed_at: z.string(),
|
|
2741
|
+
elapsed_ms: z.number().int().nonnegative(),
|
|
2742
|
+
elapsed_seconds: z.number().nonnegative()
|
|
2743
|
+
}).strict(),
|
|
2744
|
+
created_at: z.string()
|
|
2745
|
+
}).strict();
|
|
2746
|
+
var inferenceConfigSchema = z.object({
|
|
2747
|
+
maxNewTokens: z.number().int().min(1).default(256),
|
|
2748
|
+
temperature: z.number().min(0).default(0),
|
|
2749
|
+
topP: z.number().min(0).max(1).default(1),
|
|
2750
|
+
device: z.enum(["cuda", "cpu"]).default("cuda")
|
|
2751
|
+
}).strict();
|
|
2752
|
+
var scoringConfigSchema = z.object({
|
|
2753
|
+
mode: z.enum(["exact_match", "json_fields"]).default("exact_match"),
|
|
2754
|
+
fields: z.array(z.string().min(1)).optional()
|
|
2755
|
+
}).strict();
|
|
2756
|
+
var evaluationConfigSchema = z.object({
|
|
2757
|
+
inference: inferenceConfigSchema.default({
|
|
2758
|
+
maxNewTokens: 256,
|
|
2759
|
+
temperature: 0,
|
|
2760
|
+
topP: 1,
|
|
2761
|
+
device: "cuda"
|
|
2762
|
+
}),
|
|
2763
|
+
scoring: scoringConfigSchema.default({ mode: "exact_match" }),
|
|
2764
|
+
timeoutMs: z.number().int().min(100).default(18e5),
|
|
2765
|
+
maxExamples: z.number().int().min(1).optional(),
|
|
2766
|
+
sampleSeed: z.number().int().optional(),
|
|
2767
|
+
baselineCache: z.boolean().default(true),
|
|
2768
|
+
generalRegression: z.object({
|
|
2769
|
+
dataset: z.string().min(1),
|
|
2770
|
+
systemPrompt: z.string().optional(),
|
|
2771
|
+
maxScoreDrop: z.number().min(0).max(1).default(0.03),
|
|
2772
|
+
maxPassRateDrop: z.number().min(0).max(1).default(0.05)
|
|
2773
|
+
}).strict().optional()
|
|
2774
|
+
}).strict();
|
|
2775
|
+
var localRunnerConfigSchema = z.object({
|
|
2776
|
+
storeRoot: z.string().optional(),
|
|
2777
|
+
artifactRoot: z.string().default(DEFAULT_ARTIFACT_ROOT),
|
|
2778
|
+
dryRun: z.boolean().default(false),
|
|
2779
|
+
paths: z.object({
|
|
2780
|
+
baseModel: z.string().optional(),
|
|
2781
|
+
modelCache: z.string().optional()
|
|
2782
|
+
}).strict().default({}),
|
|
2783
|
+
evaluation: evaluationConfigSchema.default({
|
|
2784
|
+
inference: {
|
|
2785
|
+
maxNewTokens: 256,
|
|
2786
|
+
temperature: 0,
|
|
2787
|
+
topP: 1,
|
|
2788
|
+
device: "cuda"
|
|
2789
|
+
},
|
|
2790
|
+
scoring: { mode: "exact_match" },
|
|
2791
|
+
timeoutMs: 18e5,
|
|
2792
|
+
baselineCache: true
|
|
2793
|
+
})
|
|
2794
|
+
}).strict();
|
|
2795
|
+
function isFoundationSpecFile(spec) {
|
|
2796
|
+
return spec.engine === "foundation";
|
|
2797
|
+
}
|
|
2798
|
+
|
|
2799
|
+
// src/local-runtime/local-project.ts
|
|
2800
|
+
import { createHash as createHash3, randomUUID as randomUUID3 } from "crypto";
|
|
2801
|
+
import { readFile as readFile4, stat as stat3, writeFile as writeFile4 } from "fs/promises";
|
|
2802
|
+
import { dirname as dirname8, isAbsolute as isAbsolute3, resolve as resolve9 } from "path";
|
|
2803
|
+
import { homedir as homedir4 } from "os";
|
|
2804
|
+
import { mkdir as mkdir5 } from "fs/promises";
|
|
2805
|
+
var DEFAULT_LOCAL_SPEC_PATH = "tunedtensor.json";
|
|
2806
|
+
function resolveLocalReference(value, baseDirectory) {
|
|
2807
|
+
if (typeof value !== "string" || !value || isAbsolute3(value) || /^[a-z][a-z0-9+.-]*:/i.test(value)) {
|
|
2808
|
+
return value;
|
|
2809
|
+
}
|
|
2810
|
+
if (value === "~") return homedir4();
|
|
2811
|
+
if (value.startsWith("~/")) return resolve9(homedir4(), value.slice(2));
|
|
2812
|
+
return resolve9(baseDirectory, value);
|
|
2813
|
+
}
|
|
2814
|
+
function resolveLocalRunInputPaths(raw, inputPath) {
|
|
2815
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw;
|
|
2816
|
+
const baseDirectory = dirname8(resolve9(inputPath));
|
|
2817
|
+
const value = structuredClone(raw);
|
|
2818
|
+
const dataset = value.dataset_prebuilt;
|
|
2819
|
+
if (dataset && typeof dataset === "object" && !Array.isArray(dataset)) {
|
|
2820
|
+
const fields = dataset;
|
|
2821
|
+
for (const key of ["training", "validation", "test"]) {
|
|
2822
|
+
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
2823
|
+
}
|
|
2824
|
+
}
|
|
2825
|
+
return value;
|
|
2826
|
+
}
|
|
2827
|
+
async function exists2(path) {
|
|
2828
|
+
try {
|
|
2829
|
+
await stat3(path);
|
|
2830
|
+
return true;
|
|
2831
|
+
} catch {
|
|
2832
|
+
return false;
|
|
2833
|
+
}
|
|
2834
|
+
}
|
|
2835
|
+
async function initLocalSpecFile(args) {
|
|
2836
|
+
if (!args.force && await exists2(args.outputPath)) {
|
|
2837
|
+
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
2838
|
+
}
|
|
2839
|
+
if (args.engine === "foundation") {
|
|
2840
|
+
const spec2 = localFoundationSpecFileSchema.parse({
|
|
2841
|
+
engine: "foundation",
|
|
2842
|
+
id: randomUUID3(),
|
|
2843
|
+
name: args.name,
|
|
2844
|
+
description: "From-scratch tokenizer, pretrain, chat SFT, and eval.",
|
|
2845
|
+
system_prompt: "You are a helpful assistant.",
|
|
2846
|
+
guidelines: [
|
|
2847
|
+
"Answer the user request directly."
|
|
2848
|
+
],
|
|
2849
|
+
constraints: [],
|
|
2850
|
+
examples: [
|
|
2851
|
+
{
|
|
2852
|
+
input: "Replace this with representative chat input.",
|
|
2853
|
+
output: "Replace this with the expected assistant reply."
|
|
2854
|
+
},
|
|
2855
|
+
{
|
|
2856
|
+
input: "Replace this with a different chat input.",
|
|
2857
|
+
output: "Replace this with the expected assistant reply."
|
|
1962
2858
|
}
|
|
2859
|
+
],
|
|
2860
|
+
foundation: {
|
|
2861
|
+
depth: 2,
|
|
2862
|
+
pretrain_steps: 2,
|
|
2863
|
+
finetune_steps: 2,
|
|
2864
|
+
rl_steps: 0,
|
|
2865
|
+
vocab_size: 256,
|
|
2866
|
+
max_chars: 2e4,
|
|
2867
|
+
sequence_length: 64,
|
|
2868
|
+
batch_size: 2,
|
|
2869
|
+
nproc_per_node: 1
|
|
1963
2870
|
}
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
2871
|
+
});
|
|
2872
|
+
await mkdir5(dirname8(args.outputPath), { recursive: true });
|
|
2873
|
+
await writeFile4(args.outputPath, `${JSON.stringify(spec2, null, 2)}
|
|
2874
|
+
`, "utf8");
|
|
2875
|
+
return spec2;
|
|
2876
|
+
}
|
|
2877
|
+
if (!args.baseModel) {
|
|
2878
|
+
throw new Error("Adapter specs require --model.");
|
|
2879
|
+
}
|
|
2880
|
+
const spec = localAdapterSpecFileSchema.parse({
|
|
2881
|
+
id: randomUUID3(),
|
|
2882
|
+
name: args.name,
|
|
2883
|
+
description: "",
|
|
2884
|
+
system_prompt: "Describe the behavior this local model should learn.",
|
|
2885
|
+
guidelines: [
|
|
2886
|
+
"Return concise, task-specific answers."
|
|
2887
|
+
],
|
|
2888
|
+
constraints: [],
|
|
2889
|
+
base_model: args.baseModel,
|
|
2890
|
+
examples: [
|
|
2891
|
+
{
|
|
2892
|
+
input: "Replace this with representative training input.",
|
|
2893
|
+
output: "Replace this with the expected training output."
|
|
2894
|
+
},
|
|
2895
|
+
{
|
|
2896
|
+
input: "Replace this with a different validation input.",
|
|
2897
|
+
output: "Replace this with the expected validation output."
|
|
2898
|
+
}
|
|
2899
|
+
],
|
|
2900
|
+
hyperparameters: {
|
|
2901
|
+
n_epochs: 1
|
|
1977
2902
|
}
|
|
1978
|
-
};
|
|
2903
|
+
});
|
|
2904
|
+
await mkdir5(dirname8(args.outputPath), { recursive: true });
|
|
2905
|
+
await writeFile4(args.outputPath, `${JSON.stringify(spec, null, 2)}
|
|
2906
|
+
`, "utf8");
|
|
2907
|
+
return spec;
|
|
1979
2908
|
}
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
import { createReadStream as createReadStream2 } from "fs";
|
|
1984
|
-
import { lstat as lstat3, mkdir as mkdir5, readFile as readFile5, readdir as readdir4, realpath as realpath2, stat as stat4 } from "fs/promises";
|
|
1985
|
-
import { isAbsolute as isAbsolute3, join as join5, relative as relative2, resolve as resolve8, sep as sep3 } from "path";
|
|
1986
|
-
|
|
1987
|
-
// src/local-runtime/process-runner.ts
|
|
1988
|
-
import { createHash as createHash3 } from "crypto";
|
|
1989
|
-
import { spawn } from "child_process";
|
|
1990
|
-
import { createWriteStream, readFileSync } from "fs";
|
|
1991
|
-
import { mkdir as mkdir4 } from "fs/promises";
|
|
1992
|
-
import { homedir as homedir3 } from "os";
|
|
1993
|
-
import { dirname as dirname5, join as join3, resolve as resolve6 } from "path";
|
|
1994
|
-
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
1995
|
-
var bundledProject = join3(packageRoot, "training/local-runner");
|
|
1996
|
-
var bundledRuntimeHash = (() => {
|
|
1997
|
-
const hash = createHash3("sha256");
|
|
1998
|
-
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1999
|
-
hash.update(readFileSync(join3(bundledProject, name)));
|
|
2000
|
-
}
|
|
2001
|
-
return hash.digest("hex").slice(0, 20);
|
|
2002
|
-
})();
|
|
2003
|
-
var BUNDLED_PYTHON_ENVIRONMENT = join3(
|
|
2004
|
-
resolve6(process.env.XDG_CACHE_HOME ?? join3(homedir3(), ".cache")),
|
|
2005
|
-
"tuned-tensor-local",
|
|
2006
|
-
"uv",
|
|
2007
|
-
bundledRuntimeHash
|
|
2008
|
-
);
|
|
2009
|
-
var foundationProject = join3(packageRoot, "training/foundation");
|
|
2010
|
-
var foundationRuntimeHash = (() => {
|
|
2011
|
-
const hash = createHash3("sha256");
|
|
2012
|
-
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
2013
|
-
hash.update(readFileSync(join3(foundationProject, name)));
|
|
2909
|
+
async function initLocalRunnerConfigFile(args) {
|
|
2910
|
+
if (!args.force && await exists2(args.outputPath)) {
|
|
2911
|
+
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
2014
2912
|
}
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
"--frozen",
|
|
2028
|
-
"--quiet",
|
|
2029
|
-
"--project",
|
|
2030
|
-
bundledProject,
|
|
2031
|
-
"python",
|
|
2032
|
-
target,
|
|
2033
|
-
...args
|
|
2034
|
-
];
|
|
2035
|
-
return {
|
|
2036
|
-
command: "uv",
|
|
2037
|
-
commandArgs,
|
|
2038
|
-
displayCommand: ["uv", ...commandArgs]
|
|
2913
|
+
const config = {
|
|
2914
|
+
artifactRoot: DEFAULT_ARTIFACT_ROOT,
|
|
2915
|
+
storeRoot: DEFAULT_PROJECT_STORE_ROOT,
|
|
2916
|
+
evaluation: {
|
|
2917
|
+
inference: {
|
|
2918
|
+
device: "cuda"
|
|
2919
|
+
},
|
|
2920
|
+
scoring: {
|
|
2921
|
+
mode: "exact_match"
|
|
2922
|
+
},
|
|
2923
|
+
timeoutMs: 18e5
|
|
2924
|
+
}
|
|
2039
2925
|
};
|
|
2926
|
+
await mkdir5(dirname8(args.outputPath), { recursive: true });
|
|
2927
|
+
await writeFile4(args.outputPath, `${JSON.stringify(config, null, 2)}
|
|
2928
|
+
`, "utf8");
|
|
2929
|
+
return config;
|
|
2040
2930
|
}
|
|
2041
|
-
function
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
}
|
|
2931
|
+
function placeholderIssues(systemPrompt, examples) {
|
|
2932
|
+
const issues = [];
|
|
2933
|
+
if (/describe the behavior this local model should learn/i.test(systemPrompt)) {
|
|
2934
|
+
issues.push("system_prompt still contains the generated placeholder");
|
|
2935
|
+
}
|
|
2936
|
+
examples.forEach((example, index) => {
|
|
2937
|
+
if (/replace this with/i.test(example.input) || /replace this with/i.test(example.output)) {
|
|
2938
|
+
issues.push(`examples[${index}] still contains generated placeholder text`);
|
|
2939
|
+
}
|
|
2940
|
+
});
|
|
2941
|
+
return issues;
|
|
2046
2942
|
}
|
|
2047
|
-
function
|
|
2048
|
-
|
|
2049
|
-
"run",
|
|
2050
|
-
"--frozen",
|
|
2051
|
-
"--quiet",
|
|
2052
|
-
"--project",
|
|
2053
|
-
foundationProject,
|
|
2054
|
-
"python",
|
|
2055
|
-
join3(foundationProject, "src", entrypoint),
|
|
2056
|
-
...args
|
|
2057
|
-
];
|
|
2058
|
-
return {
|
|
2059
|
-
command: "uv",
|
|
2060
|
-
commandArgs,
|
|
2061
|
-
displayCommand: ["uv", ...commandArgs]
|
|
2062
|
-
};
|
|
2943
|
+
function generatedPlaceholderIssues(request) {
|
|
2944
|
+
return placeholderIssues(request.spec_snapshot.system_prompt, request.spec_snapshot.examples);
|
|
2063
2945
|
}
|
|
2064
|
-
function
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
}
|
|
2946
|
+
function assertLocalRunInputReady(request) {
|
|
2947
|
+
const issues = generatedPlaceholderIssues(request);
|
|
2948
|
+
if (issues.length > 0) {
|
|
2949
|
+
throw new Error(`Edit the generated behavior spec before training: ${issues.join("; ")}.`);
|
|
2950
|
+
}
|
|
2069
2951
|
}
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2952
|
+
function foundationPlaceholderIssues(spec) {
|
|
2953
|
+
return placeholderIssues(spec.system_prompt, spec.examples);
|
|
2954
|
+
}
|
|
2955
|
+
function assertFoundationSpecReady(spec) {
|
|
2956
|
+
const issues = foundationPlaceholderIssues(spec);
|
|
2957
|
+
if (issues.length > 0) {
|
|
2958
|
+
throw new Error(`Edit the generated foundation spec before training: ${issues.join("; ")}.`);
|
|
2074
2959
|
}
|
|
2075
|
-
};
|
|
2076
|
-
async function openProcessLog(path, exclusive) {
|
|
2077
|
-
const stream = createWriteStream(path, {
|
|
2078
|
-
flags: exclusive ? "wx" : "w",
|
|
2079
|
-
mode: 384
|
|
2080
|
-
});
|
|
2081
|
-
await new Promise((resolveOpen, reject) => {
|
|
2082
|
-
const onOpen = () => {
|
|
2083
|
-
stream.off("error", onError);
|
|
2084
|
-
resolveOpen();
|
|
2085
|
-
};
|
|
2086
|
-
const onError = (error) => {
|
|
2087
|
-
stream.off("open", onOpen);
|
|
2088
|
-
reject(error);
|
|
2089
|
-
};
|
|
2090
|
-
stream.once("open", onOpen);
|
|
2091
|
-
stream.once("error", onError);
|
|
2092
|
-
});
|
|
2093
|
-
return stream;
|
|
2094
2960
|
}
|
|
2095
|
-
|
|
2096
|
-
if (
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
}, args.timeoutMs) : null;
|
|
2143
|
-
const cancellationTimer = args.shouldCancel ? setInterval(() => {
|
|
2144
|
-
if (cancellationCheckRunning || cancelled) return;
|
|
2145
|
-
cancellationCheckRunning = true;
|
|
2146
|
-
Promise.resolve(args.shouldCancel?.()).then((requested) => {
|
|
2147
|
-
if (!requested || cancelled) return;
|
|
2148
|
-
cancelled = true;
|
|
2149
|
-
requestStop();
|
|
2150
|
-
}).catch((error) => {
|
|
2151
|
-
cancellationError = error;
|
|
2152
|
-
requestStop();
|
|
2153
|
-
}).finally(() => {
|
|
2154
|
-
cancellationCheckRunning = false;
|
|
2155
|
-
});
|
|
2156
|
-
}, args.cancelPollMs ?? 250) : null;
|
|
2157
|
-
cancellationTimer?.unref();
|
|
2158
|
-
const clearProcessTimers = () => {
|
|
2159
|
-
if (timer) clearTimeout(timer);
|
|
2160
|
-
if (cancellationTimer) clearInterval(cancellationTimer);
|
|
2161
|
-
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
2162
|
-
process.off("SIGINT", onSigint);
|
|
2163
|
-
process.off("SIGTERM", onSigterm);
|
|
2164
|
-
};
|
|
2165
|
-
if (logStream) {
|
|
2166
|
-
child.stdout.pipe(logStream, { end: false });
|
|
2167
|
-
child.stderr.pipe(logStream, { end: false });
|
|
2168
|
-
}
|
|
2169
|
-
forwardStreamLines(child.stdout, (line) => {
|
|
2170
|
-
args.onLine?.(line, "stdout");
|
|
2171
|
-
if (args.reporter?.verbose) {
|
|
2172
|
-
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stdout", message: line }));
|
|
2173
|
-
}
|
|
2174
|
-
});
|
|
2175
|
-
forwardStreamLines(child.stderr, (line) => {
|
|
2176
|
-
stderr += `${line}
|
|
2177
|
-
`;
|
|
2178
|
-
args.onLine?.(line, "stderr");
|
|
2179
|
-
if (args.reporter?.verbose) {
|
|
2180
|
-
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stderr", message: line }));
|
|
2181
|
-
}
|
|
2182
|
-
});
|
|
2183
|
-
child.on("error", (error) => {
|
|
2184
|
-
clearProcessTimers();
|
|
2185
|
-
reject(timedOut ? new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`) : error);
|
|
2186
|
-
});
|
|
2187
|
-
child.on("close", (code) => {
|
|
2188
|
-
if (timedOut || cancelled || cancellationError || args.terminateProcessGroupOnExit) {
|
|
2189
|
-
killProcessGroup("SIGKILL");
|
|
2190
|
-
}
|
|
2191
|
-
clearProcessTimers();
|
|
2192
|
-
if (timedOut) {
|
|
2193
|
-
reject(new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`));
|
|
2194
|
-
return;
|
|
2195
|
-
}
|
|
2196
|
-
if (cancellationError) {
|
|
2197
|
-
reject(cancellationError);
|
|
2198
|
-
return;
|
|
2199
|
-
}
|
|
2200
|
-
if (cancelled) {
|
|
2201
|
-
reject(new ProcessCancelledError(`${args.stage} was cancelled.`));
|
|
2202
|
-
return;
|
|
2203
|
-
}
|
|
2204
|
-
resolvePromise(code ?? 1);
|
|
2205
|
-
});
|
|
2206
|
-
});
|
|
2207
|
-
return { exitCode, stderr };
|
|
2208
|
-
} finally {
|
|
2209
|
-
if (logStream) {
|
|
2210
|
-
await new Promise((resolveEnd) => {
|
|
2211
|
-
logStream.end(resolveEnd);
|
|
2212
|
-
});
|
|
2213
|
-
}
|
|
2961
|
+
function runRequestFromLocalSpec(spec, options = {}) {
|
|
2962
|
+
if (isFoundationSpecFile(spec)) {
|
|
2963
|
+
throw new Error(
|
|
2964
|
+
"Foundation specs cannot be converted into a LoRA run request. Use `tt pipeline run --spec`."
|
|
2965
|
+
);
|
|
2966
|
+
}
|
|
2967
|
+
const {
|
|
2968
|
+
id,
|
|
2969
|
+
engine: _engine,
|
|
2970
|
+
hyperparameters,
|
|
2971
|
+
dataset_prebuilt: datasetPrebuilt,
|
|
2972
|
+
...specSnapshot
|
|
2973
|
+
} = spec;
|
|
2974
|
+
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
2975
|
+
return fineTuneRunRequestSchema.parse({
|
|
2976
|
+
run_id: options.runId ?? randomUUID3(),
|
|
2977
|
+
user_id: "local-user",
|
|
2978
|
+
behavior_spec_id: behaviorSpecId,
|
|
2979
|
+
run_number: 1,
|
|
2980
|
+
spec_snapshot: specSnapshot,
|
|
2981
|
+
hyperparameters,
|
|
2982
|
+
dataset_prebuilt: datasetPrebuilt
|
|
2983
|
+
});
|
|
2984
|
+
}
|
|
2985
|
+
function deterministicBehaviorSpecId(spec) {
|
|
2986
|
+
const namespace = Buffer.from("9f783f01340457bea061942004259253", "hex");
|
|
2987
|
+
const digest = createHash3("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
|
|
2988
|
+
digest[6] = digest[6] & 15 | 80;
|
|
2989
|
+
digest[8] = digest[8] & 63 | 128;
|
|
2990
|
+
const hex = digest.toString("hex");
|
|
2991
|
+
return [
|
|
2992
|
+
hex.slice(0, 8),
|
|
2993
|
+
hex.slice(8, 12),
|
|
2994
|
+
hex.slice(12, 16),
|
|
2995
|
+
hex.slice(16, 20),
|
|
2996
|
+
hex.slice(20)
|
|
2997
|
+
].join("-");
|
|
2998
|
+
}
|
|
2999
|
+
async function loadLocalRunInput(path, options = {}) {
|
|
3000
|
+
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile4(path, "utf8")), path);
|
|
3001
|
+
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
3002
|
+
if (request.success) {
|
|
3003
|
+
return { kind: "request", path, request: request.data };
|
|
3004
|
+
}
|
|
3005
|
+
const spec = localBehaviorSpecFileSchema.parse(raw);
|
|
3006
|
+
if (isFoundationSpecFile(spec)) {
|
|
3007
|
+
return { kind: "foundation-spec", path, spec };
|
|
2214
3008
|
}
|
|
3009
|
+
return {
|
|
3010
|
+
kind: "spec",
|
|
3011
|
+
path,
|
|
3012
|
+
spec,
|
|
3013
|
+
request: runRequestFromLocalSpec(spec, options)
|
|
3014
|
+
};
|
|
2215
3015
|
}
|
|
2216
3016
|
|
|
2217
|
-
// src/local-runtime/
|
|
2218
|
-
import {
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
const
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
"
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
"
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
]);
|
|
2254
|
-
const prefixes = [
|
|
2255
|
-
"LC_",
|
|
2256
|
-
"CUDA_",
|
|
2257
|
-
"NVIDIA_",
|
|
2258
|
-
"NCCL_",
|
|
2259
|
-
"OMP_",
|
|
2260
|
-
"MKL_",
|
|
2261
|
-
"TORCH_",
|
|
2262
|
-
"PYTORCH_",
|
|
2263
|
-
"TOKENIZERS_",
|
|
2264
|
-
"HF_XET_"
|
|
2265
|
-
];
|
|
2266
|
-
const result = Object.fromEntries(Object.entries(env).filter(
|
|
2267
|
-
([key]) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix))
|
|
2268
|
-
));
|
|
2269
|
-
result.TORCH_DISABLE_NATIVE_JIT ??= "1";
|
|
2270
|
-
return result;
|
|
2271
|
-
}
|
|
2272
|
-
function resolveHuggingFaceCacheLayout(modelCache) {
|
|
2273
|
-
const hfHome = resolve7(modelCache);
|
|
3017
|
+
// src/local-runtime/dataset.ts
|
|
3018
|
+
import { readFile as readFile5 } from "fs/promises";
|
|
3019
|
+
async function loadNormalizedChatJsonl(path) {
|
|
3020
|
+
const text = await readFile5(path, "utf8");
|
|
3021
|
+
const rows = [];
|
|
3022
|
+
const jsonLines = [];
|
|
3023
|
+
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
3024
|
+
if (!line.trim()) continue;
|
|
3025
|
+
let value;
|
|
3026
|
+
try {
|
|
3027
|
+
value = JSON.parse(line);
|
|
3028
|
+
} catch (error) {
|
|
3029
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: malformed JSON`, { cause: error });
|
|
3030
|
+
}
|
|
3031
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
3032
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: expected an object`);
|
|
3033
|
+
}
|
|
3034
|
+
const messages = value.messages;
|
|
3035
|
+
if (!Array.isArray(messages) || ![2, 3].includes(messages.length) || messages.some(
|
|
3036
|
+
(message) => !message || typeof message !== "object" || Array.isArray(message) || !["system", "user", "assistant"].includes(String(message.role)) || typeof message.content !== "string"
|
|
3037
|
+
)) {
|
|
3038
|
+
throw new Error(
|
|
3039
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one user message, and one assistant answer`
|
|
3040
|
+
);
|
|
3041
|
+
}
|
|
3042
|
+
const row = { messages };
|
|
3043
|
+
const offset = row.messages.length === 3 ? 1 : 0;
|
|
3044
|
+
if (offset === 1 && row.messages[0]?.role !== "system" || row.messages[offset]?.role !== "user" || !row.messages[offset]?.content.trim() || row.messages[offset + 1]?.role !== "assistant" || !row.messages[offset + 1]?.content.trim()) {
|
|
3045
|
+
throw new Error(
|
|
3046
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one non-empty user message, and one non-empty assistant answer`
|
|
3047
|
+
);
|
|
3048
|
+
}
|
|
3049
|
+
rows.push(row);
|
|
3050
|
+
jsonLines.push(JSON.stringify(row));
|
|
3051
|
+
}
|
|
3052
|
+
if (rows.length === 0) throw new Error(`Chat JSONL contains no examples: ${path}`);
|
|
2274
3053
|
return {
|
|
2275
|
-
|
|
2276
|
-
|
|
3054
|
+
rows,
|
|
3055
|
+
jsonl: jsonLines.join("\n")
|
|
2277
3056
|
};
|
|
2278
3057
|
}
|
|
2279
|
-
function
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
const
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
3058
|
+
async function normalizeChatJsonlForRelocation(path) {
|
|
3059
|
+
return (await loadNormalizedChatJsonl(path)).jsonl;
|
|
3060
|
+
}
|
|
3061
|
+
function buildSystemMessage(spec) {
|
|
3062
|
+
const parts = [];
|
|
3063
|
+
if (spec.system_prompt.trim()) parts.push(spec.system_prompt.trim());
|
|
3064
|
+
if (spec.guidelines.length > 0) {
|
|
3065
|
+
parts.push(`Guidelines:
|
|
3066
|
+
${spec.guidelines.map((guideline) => `- ${guideline}`).join("\n")}`);
|
|
3067
|
+
}
|
|
3068
|
+
if (spec.constraints.length > 0) {
|
|
3069
|
+
parts.push(`Constraints:
|
|
3070
|
+
${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
|
|
3071
|
+
}
|
|
3072
|
+
return parts.join("\n\n") || "Follow the demonstrated behavior.";
|
|
3073
|
+
}
|
|
3074
|
+
function exampleToChatRow(spec, example) {
|
|
3075
|
+
const system = buildSystemMessage(spec);
|
|
3076
|
+
return {
|
|
3077
|
+
messages: [
|
|
3078
|
+
{ role: "system", content: system },
|
|
3079
|
+
{ role: "user", content: example.input },
|
|
3080
|
+
{ role: "assistant", content: example.output }
|
|
3081
|
+
]
|
|
2288
3082
|
};
|
|
2289
|
-
delete result.TRANSFORMERS_CACHE;
|
|
2290
|
-
delete result.PYTORCH_TRANSFORMERS_CACHE;
|
|
2291
|
-
delete result.PYTORCH_PRETRAINED_BERT_CACHE;
|
|
2292
|
-
return result;
|
|
2293
3083
|
}
|
|
2294
|
-
function
|
|
3084
|
+
function compileSpecToJsonl(spec) {
|
|
3085
|
+
return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
|
|
3086
|
+
}
|
|
3087
|
+
function examplesFromSpec(spec) {
|
|
3088
|
+
return spec.examples;
|
|
3089
|
+
}
|
|
3090
|
+
async function examplesFromChatJsonl(path) {
|
|
3091
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
3092
|
+
const examples = [];
|
|
3093
|
+
for (const [index, row] of normalized.rows.entries()) {
|
|
3094
|
+
const user = row.messages.at(-2);
|
|
3095
|
+
const assistant = row.messages.at(-1);
|
|
3096
|
+
examples.push({
|
|
3097
|
+
input: user.content,
|
|
3098
|
+
output: assistant.content
|
|
3099
|
+
});
|
|
3100
|
+
}
|
|
3101
|
+
return examples;
|
|
3102
|
+
}
|
|
3103
|
+
async function evaluationSuiteFromChatJsonl(path, configuredSystem) {
|
|
3104
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
3105
|
+
const inputIdentities = /* @__PURE__ */ new Set();
|
|
3106
|
+
for (const row of normalized.rows) {
|
|
3107
|
+
const input = row.messages.at(-2).content;
|
|
3108
|
+
const identity = input.trim().replace(/\s+/g, " ").toLowerCase();
|
|
3109
|
+
if (inputIdentities.has(identity)) {
|
|
3110
|
+
throw new Error(
|
|
3111
|
+
"General regression data contains duplicate inputs; evaluation prompts must be unique."
|
|
3112
|
+
);
|
|
3113
|
+
}
|
|
3114
|
+
inputIdentities.add(identity);
|
|
3115
|
+
}
|
|
3116
|
+
const systems = new Set(
|
|
3117
|
+
normalized.rows.map((row) => row.messages.find((message) => message.role === "system")?.content).filter((value) => value !== void 0)
|
|
3118
|
+
);
|
|
3119
|
+
if (systems.size > 1) {
|
|
3120
|
+
throw new Error("General regression rows must use one consistent system message.");
|
|
3121
|
+
}
|
|
3122
|
+
const embeddedSystem = [...systems][0];
|
|
3123
|
+
if (configuredSystem !== void 0 && embeddedSystem !== void 0 && configuredSystem !== embeddedSystem) {
|
|
3124
|
+
throw new Error(
|
|
3125
|
+
"evaluation.generalRegression.systemPrompt must match the system message embedded in the dataset."
|
|
3126
|
+
);
|
|
3127
|
+
}
|
|
2295
3128
|
return {
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
3129
|
+
examples: normalized.rows.map((row) => ({
|
|
3130
|
+
input: row.messages.at(-2).content,
|
|
3131
|
+
output: row.messages.at(-1).content
|
|
3132
|
+
})),
|
|
3133
|
+
system: configuredSystem ?? embeddedSystem ?? "You are a helpful assistant."
|
|
2299
3134
|
};
|
|
2300
3135
|
}
|
|
2301
3136
|
|
|
2302
3137
|
// src/local-runtime/prefetch.ts
|
|
3138
|
+
import { createHash as createHash4 } from "crypto";
|
|
3139
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
3140
|
+
import { lstat as lstat4, mkdir as mkdir6, readFile as readFile6, readdir as readdir4, realpath as realpath2, stat as stat4 } from "fs/promises";
|
|
3141
|
+
import { isAbsolute as isAbsolute4, join as join8, relative as relative2, resolve as resolve10, sep as sep3 } from "path";
|
|
2303
3142
|
var NEMOTRON_BF16_FILE_DIGESTS = {
|
|
2304
3143
|
".gitattributes": { algorithm: "sha1", digest: "fe3f694ce14b577c9833fca6e83c5e26a7119127" },
|
|
2305
3144
|
"LICENSE": { algorithm: "sha1", digest: "4d76cc87c9edb280753e3bee3b499ae461113835" },
|
|
@@ -2366,13 +3205,13 @@ async function certifiedFileDigest(path, expected) {
|
|
|
2366
3205
|
}
|
|
2367
3206
|
function isStrictPhysicalDescendant(parent, child) {
|
|
2368
3207
|
const candidate = relative2(parent, child);
|
|
2369
|
-
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep3}`) && !
|
|
3208
|
+
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep3}`) && !isAbsolute4(candidate);
|
|
2370
3209
|
}
|
|
2371
3210
|
async function verifySnapshotCacheLocation(path, modelId, revision, modelCache) {
|
|
2372
3211
|
const repository = `models--${modelId.replaceAll("/", "--")}`;
|
|
2373
3212
|
const layout = resolveHuggingFaceCacheLayout(modelCache);
|
|
2374
|
-
const repositoryRoot =
|
|
2375
|
-
const expectedPath =
|
|
3213
|
+
const repositoryRoot = join8(layout.hubCache, repository);
|
|
3214
|
+
const expectedPath = join8(repositoryRoot, "snapshots", revision);
|
|
2376
3215
|
const [
|
|
2377
3216
|
physicalCacheRoot,
|
|
2378
3217
|
physicalHubRoot,
|
|
@@ -2384,7 +3223,7 @@ async function verifySnapshotCacheLocation(path, modelId, revision, modelCache)
|
|
|
2384
3223
|
realpath2(layout.hfHome).catch(() => null),
|
|
2385
3224
|
realpath2(layout.hubCache).catch(() => null),
|
|
2386
3225
|
realpath2(repositoryRoot).catch(() => null),
|
|
2387
|
-
realpath2(
|
|
3226
|
+
realpath2(join8(repositoryRoot, "blobs")).catch(() => null),
|
|
2388
3227
|
realpath2(path).catch(() => null),
|
|
2389
3228
|
realpath2(expectedPath).catch(() => null)
|
|
2390
3229
|
]);
|
|
@@ -2400,7 +3239,7 @@ async function verifySnapshotCacheLocation(path, modelId, revision, modelCache)
|
|
|
2400
3239
|
throw new Error(`Hugging Face repository escapes its configured hub cache: ${repositoryRoot}`);
|
|
2401
3240
|
}
|
|
2402
3241
|
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalBlobRoot)) {
|
|
2403
|
-
throw new Error(`Hugging Face blob store escapes its repository: ${
|
|
3242
|
+
throw new Error(`Hugging Face blob store escapes its repository: ${join8(repositoryRoot, "blobs")}`);
|
|
2404
3243
|
}
|
|
2405
3244
|
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalExpectedPath)) {
|
|
2406
3245
|
throw new Error(`Hugging Face snapshot escapes its configured repository: ${expectedPath}`);
|
|
@@ -2449,7 +3288,7 @@ async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelC
|
|
|
2449
3288
|
);
|
|
2450
3289
|
const physicalFiles = /* @__PURE__ */ new Map();
|
|
2451
3290
|
for (const file of snapshotFiles) {
|
|
2452
|
-
const metadata = await
|
|
3291
|
+
const metadata = await lstat4(file);
|
|
2453
3292
|
const target = await realpath2(file);
|
|
2454
3293
|
if (metadata.isSymbolicLink()) {
|
|
2455
3294
|
if (!isStrictPhysicalDescendant(physicalBlobRoot, target)) {
|
|
@@ -2462,13 +3301,13 @@ async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelC
|
|
|
2462
3301
|
const expectedFiles = new Set(Object.keys(digests));
|
|
2463
3302
|
const unexpected = [...actualFiles].filter((name) => !expectedFiles.has(name)).sort();
|
|
2464
3303
|
if (unexpected.length > 0) {
|
|
2465
|
-
throw new Error(`Certified snapshot contains unexpected file: ${
|
|
3304
|
+
throw new Error(`Certified snapshot contains unexpected file: ${join8(path, unexpected[0])}`);
|
|
2466
3305
|
}
|
|
2467
3306
|
for (const [name, expected] of Object.entries(digests)) {
|
|
2468
3307
|
if (!actualFiles.has(name)) {
|
|
2469
|
-
throw new Error(`Certified snapshot file is missing: ${
|
|
3308
|
+
throw new Error(`Certified snapshot file is missing: ${join8(path, name)}`);
|
|
2470
3309
|
}
|
|
2471
|
-
const file =
|
|
3310
|
+
const file = join8(path, name);
|
|
2472
3311
|
const physicalFile = physicalFiles.get(file);
|
|
2473
3312
|
if (await certifiedFileDigest(physicalFile, expected) !== expected.digest) {
|
|
2474
3313
|
throw new Error(`Certified snapshot file checksum mismatch: ${file}`);
|
|
@@ -2487,7 +3326,7 @@ function buildModelPrefetchPayload(request, config) {
|
|
|
2487
3326
|
return {
|
|
2488
3327
|
base_model: model.id,
|
|
2489
3328
|
...revision ? { revision } : {},
|
|
2490
|
-
...config.paths.modelCache ? { model_cache:
|
|
3329
|
+
...config.paths.modelCache ? { model_cache: resolve10(config.paths.modelCache) } : {}
|
|
2491
3330
|
};
|
|
2492
3331
|
}
|
|
2493
3332
|
function safeName(value) {
|
|
@@ -2498,7 +3337,7 @@ function transformersWeightName(name) {
|
|
|
2498
3337
|
return lower.endsWith(".safetensors") && !lower.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(lower);
|
|
2499
3338
|
}
|
|
2500
3339
|
async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
2501
|
-
const root = await
|
|
3340
|
+
const root = await lstat4(path).catch(() => null);
|
|
2502
3341
|
if (!root) throw new Error(`paths.baseModel is set to ${path}, but that path does not exist.`);
|
|
2503
3342
|
if (root.isSymbolicLink()) {
|
|
2504
3343
|
throw new Error(`paths.baseModel must not itself be a symbolic link: ${path}`);
|
|
@@ -2511,7 +3350,7 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2511
3350
|
const files = [];
|
|
2512
3351
|
const visit = async (directory) => {
|
|
2513
3352
|
for (const entry of await readdir4(directory, { withFileTypes: true })) {
|
|
2514
|
-
const child =
|
|
3353
|
+
const child = join8(directory, entry.name);
|
|
2515
3354
|
if (entry.isSymbolicLink()) {
|
|
2516
3355
|
const metadata2 = await stat4(child);
|
|
2517
3356
|
if (!metadata2.isFile()) {
|
|
@@ -2535,15 +3374,15 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2535
3374
|
throw new Error(`Local base-model directory contains no non-empty Transformers model weights: ${path}`);
|
|
2536
3375
|
}
|
|
2537
3376
|
const required = async (name) => {
|
|
2538
|
-
const metadata = await stat4(
|
|
3377
|
+
const metadata = await stat4(join8(path, name)).catch(() => null);
|
|
2539
3378
|
return Boolean(metadata?.isFile() && metadata.size > 0);
|
|
2540
3379
|
};
|
|
2541
3380
|
if (!await required("config.json")) throw new Error(`Local base-model directory is missing config.json: ${path}`);
|
|
2542
3381
|
try {
|
|
2543
|
-
const config = JSON.parse(await
|
|
3382
|
+
const config = JSON.parse(await readFile6(join8(path, "config.json"), "utf8"));
|
|
2544
3383
|
assertCertifiedBaseModelConfig(
|
|
2545
3384
|
config,
|
|
2546
|
-
`Local base-model config ${
|
|
3385
|
+
`Local base-model config ${join8(path, "config.json")}`,
|
|
2547
3386
|
expectedModelId
|
|
2548
3387
|
);
|
|
2549
3388
|
} catch (error) {
|
|
@@ -2573,7 +3412,7 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2573
3412
|
throw new Error(`Local base-model directory is missing tokenizer metadata or vocabulary: ${path}`);
|
|
2574
3413
|
}
|
|
2575
3414
|
for (const file of files.filter((candidate) => candidate.path.endsWith(".index.json"))) {
|
|
2576
|
-
const parsed = JSON.parse(await
|
|
3415
|
+
const parsed = JSON.parse(await readFile6(file.path, "utf8"));
|
|
2577
3416
|
if (!parsed.weight_map || typeof parsed.weight_map !== "object" || Array.isArray(parsed.weight_map)) {
|
|
2578
3417
|
throw new Error(`Local base-model weight index is invalid: ${file.path}`);
|
|
2579
3418
|
}
|
|
@@ -2593,14 +3432,14 @@ async function prefetchBaseModel(args) {
|
|
|
2593
3432
|
...buildModelPrefetchPayload(args.request, args.config),
|
|
2594
3433
|
...args.localOnly ? { local_files_only: true } : {}
|
|
2595
3434
|
};
|
|
2596
|
-
const artifactDir =
|
|
3435
|
+
const artifactDir = resolve10(
|
|
2597
3436
|
args.config.artifactRoot,
|
|
2598
3437
|
args.localOnly ? "verify-base" : "prefetch",
|
|
2599
3438
|
`${safeName(args.request.spec_snapshot.base_model)}-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
|
|
2600
3439
|
);
|
|
2601
|
-
await
|
|
3440
|
+
await mkdir6(artifactDir, { recursive: true });
|
|
2602
3441
|
if (args.config.paths.baseModel) {
|
|
2603
|
-
const localPath =
|
|
3442
|
+
const localPath = resolve10(args.config.paths.baseModel);
|
|
2604
3443
|
const verified = await verifyLocalBaseModel(
|
|
2605
3444
|
localPath,
|
|
2606
3445
|
args.request.spec_snapshot.base_model,
|
|
@@ -2617,9 +3456,9 @@ async function prefetchBaseModel(args) {
|
|
|
2617
3456
|
reason: args.localOnly ? "Verified the configured local base-model artifact; no network access was used." : "paths.baseModel points at a verified local base-model artifact; no Hugging Face download is needed."
|
|
2618
3457
|
};
|
|
2619
3458
|
}
|
|
2620
|
-
const inputPath =
|
|
2621
|
-
const outputPath =
|
|
2622
|
-
const logPath =
|
|
3459
|
+
const inputPath = join8(artifactDir, "prefetch-input.json");
|
|
3460
|
+
const outputPath = join8(artifactDir, "prefetch-output.json");
|
|
3461
|
+
const logPath = join8(artifactDir, "prefetch.log");
|
|
2623
3462
|
await writeJson(inputPath, payload);
|
|
2624
3463
|
const entrypoint = buildBundledPythonCommand(
|
|
2625
3464
|
"prefetch.py",
|
|
@@ -2656,7 +3495,7 @@ async function prefetchBaseModel(args) {
|
|
|
2656
3495
|
if (exitCode !== 0) {
|
|
2657
3496
|
throw new Error(`Model prefetch exited with code ${exitCode}. See ${logPath}.`);
|
|
2658
3497
|
}
|
|
2659
|
-
const output = JSON.parse(await
|
|
3498
|
+
const output = JSON.parse(await readFile6(outputPath, "utf8"));
|
|
2660
3499
|
const snapshotRevision = baseModelRevisionSchema.safeParse(output.snapshot_revision);
|
|
2661
3500
|
if (!snapshotRevision.success) {
|
|
2662
3501
|
throw new Error(
|
|
@@ -2739,17 +3578,17 @@ function evaluateGeneralRegressionGate(comparison, policy) {
|
|
|
2739
3578
|
}
|
|
2740
3579
|
|
|
2741
3580
|
// src/local-runtime/orchestrator.ts
|
|
2742
|
-
import { lstat as
|
|
3581
|
+
import { lstat as lstat5, mkdir as mkdir9, open as open2, readFile as readFile9, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
|
|
2743
3582
|
import { createReadStream as createReadStream3 } from "fs";
|
|
2744
3583
|
import { createHash as createHash6, randomUUID as randomUUID4 } from "crypto";
|
|
2745
3584
|
import { performance } from "perf_hooks";
|
|
2746
3585
|
import { homedir as homedir5 } from "os";
|
|
2747
|
-
import { basename, dirname as
|
|
3586
|
+
import { basename, dirname as dirname10, join as join11, relative as relative3, resolve as resolve13 } from "path";
|
|
2748
3587
|
|
|
2749
3588
|
// src/local-runtime/evaluation.ts
|
|
2750
3589
|
import { createHash as createHash5 } from "crypto";
|
|
2751
|
-
import { mkdir as
|
|
2752
|
-
import { dirname as
|
|
3590
|
+
import { mkdir as mkdir7, readFile as readFile7, rm as rm2, writeFile as writeFile5 } from "fs/promises";
|
|
3591
|
+
import { dirname as dirname9, join as join9, resolve as resolve11 } from "path";
|
|
2753
3592
|
function normalize(value) {
|
|
2754
3593
|
return value.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2755
3594
|
}
|
|
@@ -2969,8 +3808,8 @@ async function runTransformersInference(args) {
|
|
|
2969
3808
|
const inputPath = `${args.outputPath}.inference-input.json`;
|
|
2970
3809
|
const outputPath = `${args.outputPath}.inference-output.json`;
|
|
2971
3810
|
const logPath = `${args.outputPath}.inference.log`;
|
|
2972
|
-
await
|
|
2973
|
-
await
|
|
3811
|
+
await mkdir7(dirname9(inputPath), { recursive: true });
|
|
3812
|
+
await writeFile5(inputPath, `${JSON.stringify({
|
|
2974
3813
|
protocol_version: INFERENCE_PROTOCOL_VERSION,
|
|
2975
3814
|
kind: args.kind,
|
|
2976
3815
|
model_id: args.modelId,
|
|
@@ -2984,7 +3823,7 @@ async function runTransformersInference(args) {
|
|
|
2984
3823
|
id: String(index),
|
|
2985
3824
|
input: example.input
|
|
2986
3825
|
})),
|
|
2987
|
-
model_cache: args.config.paths.modelCache ?
|
|
3826
|
+
model_cache: args.config.paths.modelCache ? resolve11(args.config.paths.modelCache) : void 0,
|
|
2988
3827
|
trust_remote_code: false,
|
|
2989
3828
|
device: args.config.evaluation.inference.device,
|
|
2990
3829
|
generation: {
|
|
@@ -3033,7 +3872,7 @@ async function runTransformersInference(args) {
|
|
|
3033
3872
|
}
|
|
3034
3873
|
let output;
|
|
3035
3874
|
try {
|
|
3036
|
-
output = JSON.parse(await
|
|
3875
|
+
output = JSON.parse(await readFile7(outputPath, "utf8"));
|
|
3037
3876
|
} catch (error) {
|
|
3038
3877
|
throw new Error(
|
|
3039
3878
|
`Transformers inference did not write valid JSON: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -3069,13 +3908,16 @@ function baselineCacheKey(args) {
|
|
|
3069
3908
|
function baselineInputsAreStable(args) {
|
|
3070
3909
|
return args.config.paths.baseModel ? Boolean(args.sourceFingerprint) : Boolean(args.baseModelRevision);
|
|
3071
3910
|
}
|
|
3911
|
+
function baselineStoreRoot(config) {
|
|
3912
|
+
return config.storeRoot ? resolve11(config.storeRoot) : defaultLocalHome();
|
|
3913
|
+
}
|
|
3072
3914
|
function baselineCachePath(config, key) {
|
|
3073
|
-
|
|
3074
|
-
return join6(root, "cache", "baseline-evals", `${key}.json`);
|
|
3915
|
+
return join9(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
|
|
3075
3916
|
}
|
|
3076
|
-
async function readBaselineCache(path) {
|
|
3917
|
+
async function readBaselineCache(config, path) {
|
|
3918
|
+
await hardenExistingLocalStore(baselineStoreRoot(config));
|
|
3077
3919
|
try {
|
|
3078
|
-
return evalReportSchema.parse(JSON.parse(await
|
|
3920
|
+
return evalReportSchema.parse(JSON.parse(await readFile7(path, "utf8")));
|
|
3079
3921
|
} catch {
|
|
3080
3922
|
return null;
|
|
3081
3923
|
}
|
|
@@ -3084,7 +3926,7 @@ var cachedPackageVersion = null;
|
|
|
3084
3926
|
async function packageVersion() {
|
|
3085
3927
|
if (cachedPackageVersion !== null) return cachedPackageVersion;
|
|
3086
3928
|
try {
|
|
3087
|
-
const raw = await
|
|
3929
|
+
const raw = await readFile7(resolve11(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
|
|
3088
3930
|
cachedPackageVersion = String(
|
|
3089
3931
|
JSON.parse(raw).version ?? "unknown"
|
|
3090
3932
|
);
|
|
@@ -3153,7 +3995,7 @@ async function evaluateExamples(args) {
|
|
|
3153
3995
|
baseModelId,
|
|
3154
3996
|
args.baseModelRevision
|
|
3155
3997
|
);
|
|
3156
|
-
const localModelSource = args.modelSource ?
|
|
3998
|
+
const localModelSource = args.modelSource ? resolve11(args.modelSource) !== resolve11(baseModelId) : false;
|
|
3157
3999
|
const baseModelRevision = localModelSource ? defaultBaseModelRevision(baseModelId) : requestedBaseModelRevision;
|
|
3158
4000
|
const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
|
|
3159
4001
|
const truncated = args.examples.length > maxExamples;
|
|
@@ -3178,7 +4020,7 @@ async function evaluateExamples(args) {
|
|
|
3178
4020
|
packageVersion: await packageVersion()
|
|
3179
4021
|
}) : null;
|
|
3180
4022
|
if (cacheKey) {
|
|
3181
|
-
const cached = await readBaselineCache(baselineCachePath(args.config, cacheKey));
|
|
4023
|
+
const cached = await readBaselineCache(args.config, baselineCachePath(args.config, cacheKey));
|
|
3182
4024
|
if (cached) {
|
|
3183
4025
|
const report2 = evalReportSchema.parse({
|
|
3184
4026
|
...cached,
|
|
@@ -3229,7 +4071,8 @@ async function evaluateExamples(args) {
|
|
|
3229
4071
|
logUri: inference ? fileUri(`${args.outputPath}.inference.log`) : void 0
|
|
3230
4072
|
});
|
|
3231
4073
|
if (cacheKey) {
|
|
3232
|
-
await
|
|
4074
|
+
await createLocalStore(baselineStoreRoot(args.config)).ensure();
|
|
4075
|
+
await writePrivateJsonAtomic(
|
|
3233
4076
|
baselineCachePath(args.config, cacheKey),
|
|
3234
4077
|
{ ...report, cache_key: cacheKey }
|
|
3235
4078
|
);
|
|
@@ -3268,8 +4111,8 @@ function compareEvalReports(baseline, candidate) {
|
|
|
3268
4111
|
}
|
|
3269
4112
|
|
|
3270
4113
|
// src/local-runtime/process-training.ts
|
|
3271
|
-
import { access, mkdir as
|
|
3272
|
-
import { join as
|
|
4114
|
+
import { access as access2, mkdir as mkdir8, readFile as readFile8, writeFile as writeFile6 } from "fs/promises";
|
|
4115
|
+
import { join as join10, resolve as resolve12 } from "path";
|
|
3273
4116
|
function buildTrainingHyperparameters(request, options = {}) {
|
|
3274
4117
|
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
3275
4118
|
const hyper = request.hyperparameters;
|
|
@@ -3293,7 +4136,7 @@ function buildTrainingHyperparameters(request, options = {}) {
|
|
|
3293
4136
|
}
|
|
3294
4137
|
async function pathExists(path) {
|
|
3295
4138
|
try {
|
|
3296
|
-
await
|
|
4139
|
+
await access2(path);
|
|
3297
4140
|
return true;
|
|
3298
4141
|
} catch {
|
|
3299
4142
|
return false;
|
|
@@ -3360,15 +4203,15 @@ function createTrainingProgressForwarder(reporter) {
|
|
|
3360
4203
|
async function launchProcessTraining(args) {
|
|
3361
4204
|
const { request, artifacts, config } = args;
|
|
3362
4205
|
const jobName = `tt-local-${request.run_id}`;
|
|
3363
|
-
await
|
|
4206
|
+
await mkdir8(artifacts.trainingDir, { recursive: true });
|
|
3364
4207
|
await copyDatasetToTrainingChannel(artifacts);
|
|
3365
4208
|
const hyperparameters = buildTrainingHyperparameters(request, {
|
|
3366
4209
|
baseModelRevision: args.baseModelRevision
|
|
3367
4210
|
});
|
|
3368
|
-
await writeJson(
|
|
4211
|
+
await writeJson(join10(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
|
|
3369
4212
|
const entrypoint = buildBundledPythonCommand("train.py");
|
|
3370
4213
|
if (config.dryRun) {
|
|
3371
|
-
await
|
|
4214
|
+
await writeFile6(
|
|
3372
4215
|
artifacts.trainingLog,
|
|
3373
4216
|
"Dry run enabled. Training process was not launched.\n",
|
|
3374
4217
|
"utf8"
|
|
@@ -3390,20 +4233,20 @@ async function launchProcessTraining(args) {
|
|
|
3390
4233
|
withOfflineHuggingFaceCacheEnvironment({
|
|
3391
4234
|
...inheritedEnv,
|
|
3392
4235
|
BACKEND: "local",
|
|
3393
|
-
SM_CHANNEL_TRAINING:
|
|
3394
|
-
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ?
|
|
3395
|
-
SM_MODEL_DIR:
|
|
3396
|
-
SM_OUTPUT_DIR:
|
|
3397
|
-
TT_HYPERPARAMETERS_PATH:
|
|
4236
|
+
SM_CHANNEL_TRAINING: resolve12(artifacts.trainingInputDir),
|
|
4237
|
+
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve12(config.paths.baseModel) : void 0,
|
|
4238
|
+
SM_MODEL_DIR: resolve12(artifacts.trainingModelDir),
|
|
4239
|
+
SM_OUTPUT_DIR: resolve12(artifacts.trainingOutputDir),
|
|
4240
|
+
TT_HYPERPARAMETERS_PATH: resolve12(artifacts.trainingConfigDir, "hyperparameters.json")
|
|
3398
4241
|
}, config.paths.modelCache)
|
|
3399
4242
|
);
|
|
3400
4243
|
for (const [key, value] of Object.entries(childEnv)) {
|
|
3401
4244
|
if (value === void 0) delete childEnv[key];
|
|
3402
4245
|
}
|
|
3403
|
-
if (config.paths.modelCache) await
|
|
4246
|
+
if (config.paths.modelCache) await mkdir8(resolve12(config.paths.modelCache), { recursive: true });
|
|
3404
4247
|
if (config.paths.baseModel) {
|
|
3405
4248
|
await verifyLocalBaseModel(
|
|
3406
|
-
|
|
4249
|
+
resolve12(config.paths.baseModel),
|
|
3407
4250
|
request.spec_snapshot.base_model,
|
|
3408
4251
|
config.paths.modelCache
|
|
3409
4252
|
);
|
|
@@ -3430,12 +4273,12 @@ async function launchProcessTraining(args) {
|
|
|
3430
4273
|
onLine: (line) => forwardTrainingProgress(line),
|
|
3431
4274
|
shouldCancel: args.shouldCancel
|
|
3432
4275
|
});
|
|
3433
|
-
const metricsPath =
|
|
3434
|
-
const metrics = await pathExists(metricsPath) ? JSON.parse(await
|
|
4276
|
+
const metricsPath = join10(artifacts.trainingModelDir, "training-metrics.json");
|
|
4277
|
+
const metrics = await pathExists(metricsPath) ? JSON.parse(await readFile8(metricsPath, "utf8")) : null;
|
|
3435
4278
|
if (exitCode !== 0) {
|
|
3436
4279
|
throw new Error(`Training process exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
|
|
3437
4280
|
}
|
|
3438
|
-
const modelTarPath =
|
|
4281
|
+
const modelTarPath = join10(artifacts.trainingOutputDir, "model.tar.gz");
|
|
3439
4282
|
const modelUri = await pathExists(modelTarPath) ? fileUri(modelTarPath) : fileUri(artifacts.trainingModelDir);
|
|
3440
4283
|
await args.reporter?.onEvent?.({
|
|
3441
4284
|
stage: "training",
|
|
@@ -3530,21 +4373,21 @@ function validateLocalPipeline(pipeline) {
|
|
|
3530
4373
|
return pipeline;
|
|
3531
4374
|
}
|
|
3532
4375
|
async function loadJsonFile(path) {
|
|
3533
|
-
return JSON.parse(await
|
|
4376
|
+
return JSON.parse(await readFile9(path, "utf8"));
|
|
3534
4377
|
}
|
|
3535
4378
|
async function loadRunRequest(path) {
|
|
3536
4379
|
return fineTuneRunRequestSchema.parse(await loadJsonFile(path));
|
|
3537
4380
|
}
|
|
3538
4381
|
async function loadLocalRunnerConfig(path) {
|
|
3539
4382
|
if (!path) return localRunnerConfigSchema.parse({});
|
|
3540
|
-
const configPath =
|
|
3541
|
-
const base =
|
|
4383
|
+
const configPath = resolve13(path);
|
|
4384
|
+
const base = dirname10(configPath);
|
|
3542
4385
|
const config = localRunnerConfigSchema.parse(await loadJsonFile(configPath));
|
|
3543
4386
|
const configPathValue = (value) => {
|
|
3544
4387
|
if (!value) return void 0;
|
|
3545
4388
|
if (value === "~") return homedir5();
|
|
3546
|
-
if (value.startsWith("~/")) return
|
|
3547
|
-
return
|
|
4389
|
+
if (value.startsWith("~/")) return resolve13(homedir5(), value.slice(2));
|
|
4390
|
+
return resolve13(base, value);
|
|
3548
4391
|
};
|
|
3549
4392
|
return {
|
|
3550
4393
|
...config,
|
|
@@ -3742,21 +4585,21 @@ async function hashFileIfPresent(path) {
|
|
|
3742
4585
|
}
|
|
3743
4586
|
async function packageVersion2() {
|
|
3744
4587
|
try {
|
|
3745
|
-
const parsed = JSON.parse(await
|
|
4588
|
+
const parsed = JSON.parse(await readFile9(resolve13(packageRoot2, "package.json"), "utf8"));
|
|
3746
4589
|
return typeof parsed.version === "string" ? parsed.version : "unknown";
|
|
3747
4590
|
} catch {
|
|
3748
4591
|
return "unknown";
|
|
3749
4592
|
}
|
|
3750
4593
|
}
|
|
3751
4594
|
async function runtimeFingerprint() {
|
|
3752
|
-
const bundledProject2 =
|
|
4595
|
+
const bundledProject2 = resolve13(packageRoot2, "training/adapter");
|
|
3753
4596
|
return hashJson({
|
|
3754
4597
|
tt_local_version: await packageVersion2(),
|
|
3755
4598
|
node_version: process.version,
|
|
3756
4599
|
platform: process.platform,
|
|
3757
4600
|
architecture: process.arch,
|
|
3758
|
-
runner_pyproject: await hashFileIfPresent(
|
|
3759
|
-
runner_uv_lock: await hashFileIfPresent(
|
|
4601
|
+
runner_pyproject: await hashFileIfPresent(resolve13(bundledProject2, "pyproject.toml")),
|
|
4602
|
+
runner_uv_lock: await hashFileIfPresent(resolve13(bundledProject2, "uv.lock"))
|
|
3760
4603
|
});
|
|
3761
4604
|
}
|
|
3762
4605
|
function preparedSourceFingerprint(args) {
|
|
@@ -3803,9 +4646,9 @@ async function resolveBaseModelRevision(request, config) {
|
|
|
3803
4646
|
}
|
|
3804
4647
|
const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
|
|
3805
4648
|
const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
|
|
3806
|
-
const refPath =
|
|
4649
|
+
const refPath = resolve13(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
|
|
3807
4650
|
try {
|
|
3808
|
-
const revision = (await
|
|
4651
|
+
const revision = (await readFile9(refPath, "utf8")).trim();
|
|
3809
4652
|
if (revision) return revision;
|
|
3810
4653
|
} catch {
|
|
3811
4654
|
}
|
|
@@ -3813,7 +4656,7 @@ async function resolveBaseModelRevision(request, config) {
|
|
|
3813
4656
|
}
|
|
3814
4657
|
async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
|
|
3815
4658
|
const root = localModelArtifactPath(uri);
|
|
3816
|
-
const rootMetadata = await
|
|
4659
|
+
const rootMetadata = await lstat5(root);
|
|
3817
4660
|
if (rootMetadata.isSymbolicLink()) {
|
|
3818
4661
|
throw new Error(`Local base model path must not itself be a symbolic link: ${root}`);
|
|
3819
4662
|
}
|
|
@@ -3824,7 +4667,7 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
|
|
|
3824
4667
|
const files = [];
|
|
3825
4668
|
const visit = async (directory) => {
|
|
3826
4669
|
for (const entry of await readdir5(directory, { withFileTypes: true })) {
|
|
3827
|
-
const child =
|
|
4670
|
+
const child = resolve13(directory, entry.name);
|
|
3828
4671
|
if (entry.isDirectory()) {
|
|
3829
4672
|
await visit(child);
|
|
3830
4673
|
continue;
|
|
@@ -3863,12 +4706,12 @@ async function readStageMetadata(path) {
|
|
|
3863
4706
|
}
|
|
3864
4707
|
}
|
|
3865
4708
|
async function removePrefixedArtifacts(path) {
|
|
3866
|
-
const directory =
|
|
4709
|
+
const directory = dirname10(path);
|
|
3867
4710
|
const prefix = `${basename(path)}.`;
|
|
3868
4711
|
const names = await readdir5(directory).catch(() => []);
|
|
3869
4712
|
await Promise.all([
|
|
3870
4713
|
rm3(path, { force: true }),
|
|
3871
|
-
...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(
|
|
4714
|
+
...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(resolve13(directory, name), { recursive: true, force: true }))
|
|
3872
4715
|
]);
|
|
3873
4716
|
}
|
|
3874
4717
|
async function cleanupStageArtifacts(artifacts, stage) {
|
|
@@ -3881,7 +4724,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
3881
4724
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
3882
4725
|
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
3883
4726
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
3884
|
-
rm3(
|
|
4727
|
+
rm3(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
3885
4728
|
removeReport(),
|
|
3886
4729
|
rm3(artifacts.artifactManifestJson, { force: true })
|
|
3887
4730
|
]);
|
|
@@ -3899,7 +4742,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
3899
4742
|
await Promise.all([
|
|
3900
4743
|
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
3901
4744
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
3902
|
-
rm3(
|
|
4745
|
+
rm3(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
3903
4746
|
removePrefixedArtifacts(artifacts.candidateEvalJson),
|
|
3904
4747
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
3905
4748
|
removeReport()
|
|
@@ -3935,7 +4778,7 @@ async function throwIfCancelled(store, request) {
|
|
|
3935
4778
|
}
|
|
3936
4779
|
}
|
|
3937
4780
|
async function acquireWorkflowLock(lockPath, description) {
|
|
3938
|
-
await
|
|
4781
|
+
await mkdir9(dirname10(lockPath), { recursive: true });
|
|
3939
4782
|
const token = randomUUID4();
|
|
3940
4783
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
3941
4784
|
try {
|
|
@@ -3948,7 +4791,7 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
3948
4791
|
}
|
|
3949
4792
|
return async () => {
|
|
3950
4793
|
try {
|
|
3951
|
-
const owner = JSON.parse(await
|
|
4794
|
+
const owner = JSON.parse(await readFile9(lockPath, "utf8"));
|
|
3952
4795
|
if (owner.token === token) await rm3(lockPath, { force: true });
|
|
3953
4796
|
} catch {
|
|
3954
4797
|
}
|
|
@@ -3958,7 +4801,7 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
3958
4801
|
const metadata = await stat5(lockPath).catch(() => null);
|
|
3959
4802
|
let owner = null;
|
|
3960
4803
|
try {
|
|
3961
|
-
owner = JSON.parse(await
|
|
4804
|
+
owner = JSON.parse(await readFile9(lockPath, "utf8"));
|
|
3962
4805
|
} catch {
|
|
3963
4806
|
owner = null;
|
|
3964
4807
|
}
|
|
@@ -3989,11 +4832,12 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
3989
4832
|
throw new Error(`Unable to acquire local workflow lock for ${description}.`);
|
|
3990
4833
|
}
|
|
3991
4834
|
async function acquireRunLock(store, runId) {
|
|
3992
|
-
|
|
4835
|
+
await store.ensure();
|
|
4836
|
+
return acquireWorkflowLock(resolve13(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
|
|
3993
4837
|
}
|
|
3994
4838
|
async function acquireArtifactLock(artifacts) {
|
|
3995
4839
|
return acquireWorkflowLock(
|
|
3996
|
-
|
|
4840
|
+
join11(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
|
|
3997
4841
|
`Artifact directory ${artifacts.runDir}`
|
|
3998
4842
|
);
|
|
3999
4843
|
}
|
|
@@ -4040,13 +4884,13 @@ function stageFingerprintPath(prepared, stage) {
|
|
|
4040
4884
|
return `${stageOutputPath(prepared, stage)}.stage.json`;
|
|
4041
4885
|
}
|
|
4042
4886
|
async function runnerFileFingerprints() {
|
|
4043
|
-
const project =
|
|
4044
|
-
const bundledSource =
|
|
4045
|
-
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) =>
|
|
4887
|
+
const project = resolve13(packageRoot2, "training/adapter");
|
|
4888
|
+
const bundledSource = resolve13(packageRoot2, "training/adapter/src");
|
|
4889
|
+
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve13(bundledSource, entry.name));
|
|
4046
4890
|
const candidates = [.../* @__PURE__ */ new Set([
|
|
4047
4891
|
...pythonSources,
|
|
4048
|
-
|
|
4049
|
-
|
|
4892
|
+
resolve13(project, "pyproject.toml"),
|
|
4893
|
+
resolve13(project, "uv.lock")
|
|
4050
4894
|
])];
|
|
4051
4895
|
const fingerprints = {};
|
|
4052
4896
|
for (const path of candidates.sort()) fingerprints[path] = await hashFileIfPresent(path);
|
|
@@ -4160,7 +5004,7 @@ async function ensureRunRecord(args) {
|
|
|
4160
5004
|
`Run ${args.request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
|
|
4161
5005
|
);
|
|
4162
5006
|
}
|
|
4163
|
-
if (
|
|
5007
|
+
if (resolve13(existing.artifact_dir) !== resolve13(args.artifacts.runDir)) {
|
|
4164
5008
|
throw new Error(
|
|
4165
5009
|
`Run ${args.request.run_id} is already bound to artifact directory ${existing.artifact_dir}; it cannot be resumed with ${args.artifacts.runDir}.`
|
|
4166
5010
|
);
|
|
@@ -4267,7 +5111,7 @@ async function prepareStage(args) {
|
|
|
4267
5111
|
const preparedExists = await pathExists2(args.artifacts.stageMetadataJson) && await pathExists2(args.artifacts.trainingJsonl);
|
|
4268
5112
|
const prepared = await computePreparedRun({ ...args, writeArtifacts: false });
|
|
4269
5113
|
await args.store.syncRunRequest(args.request, args.artifacts.runDir);
|
|
4270
|
-
await writeJsonAtomic(
|
|
5114
|
+
await writeJsonAtomic(resolve13(args.artifacts.runDir, "request.json"), args.request);
|
|
4271
5115
|
const existingMetadata = preparedExists ? await readStageMetadata(args.artifacts.stageMetadataJson) : null;
|
|
4272
5116
|
let canReuse = preparedExists && existingMetadata?.source_fingerprint === prepared.metadata.source_fingerprint;
|
|
4273
5117
|
if (canReuse) {
|
|
@@ -4796,7 +5640,7 @@ async function runLocalPipeline(input) {
|
|
|
4796
5640
|
if (step.uses === "evaluate") {
|
|
4797
5641
|
const isBase = step.with.model === "base";
|
|
4798
5642
|
const report = isBase ? await runBaselineStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter }) : await runCandidateStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
|
|
4799
|
-
const reportPath =
|
|
5643
|
+
const reportPath = join11(artifacts.runDir, "evaluations", `${step.id}.json`);
|
|
4800
5644
|
await writeJsonAtomic(reportPath, { ...report, artifact_uri: fileUri(reportPath) });
|
|
4801
5645
|
outputs[step.id] = { report: { ...report, artifact_uri: fileUri(reportPath) }, reportPath };
|
|
4802
5646
|
await refreshArtifactManifest(prepared);
|
|
@@ -4872,7 +5716,6 @@ async function runLocalFineTune(input) {
|
|
|
4872
5716
|
}
|
|
4873
5717
|
|
|
4874
5718
|
export {
|
|
4875
|
-
writeJsonAtomic,
|
|
4876
5719
|
assertArtifactManifest,
|
|
4877
5720
|
TRAINING_MODELS,
|
|
4878
5721
|
resolveTrainingModel,
|
|
@@ -4880,6 +5723,35 @@ export {
|
|
|
4880
5723
|
defaultBaseModelRevision,
|
|
4881
5724
|
resolveModelLoader,
|
|
4882
5725
|
assertUsableModelArtifact,
|
|
5726
|
+
DEFAULT_ARTIFACT_ROOT,
|
|
5727
|
+
DEFAULT_FOUNDATION_RUNS_DIR,
|
|
5728
|
+
expandUserPath,
|
|
5729
|
+
getConfigDir,
|
|
5730
|
+
getAgentConfigDir,
|
|
5731
|
+
defaultStoreRoot,
|
|
5732
|
+
warningsFromSnapshot,
|
|
5733
|
+
minimalMachineLearningEnvironment,
|
|
5734
|
+
withHuggingFaceCacheEnvironment,
|
|
5735
|
+
withOfflineHuggingFaceCacheEnvironment,
|
|
5736
|
+
reportInBackground,
|
|
5737
|
+
sanitizeLogLine,
|
|
5738
|
+
forwardStreamLines,
|
|
5739
|
+
localRuntimePackageRoot,
|
|
5740
|
+
buildBundledPythonCommand,
|
|
5741
|
+
withBundledPythonEnvironment,
|
|
5742
|
+
buildFoundationPythonCommand,
|
|
5743
|
+
withFoundationPythonEnvironment,
|
|
5744
|
+
runLoggedProcess,
|
|
5745
|
+
readHardwareSnapshot,
|
|
5746
|
+
formatHostStatusLine,
|
|
5747
|
+
formatAgentHostBlock,
|
|
5748
|
+
assessHardware,
|
|
5749
|
+
isTerminalRunState,
|
|
5750
|
+
defaultLocalHome,
|
|
5751
|
+
localStorePaths,
|
|
5752
|
+
hardenExistingLocalStore,
|
|
5753
|
+
writePrivateJsonAtomic,
|
|
5754
|
+
createLocalStore,
|
|
4883
5755
|
datasetFormatSchema,
|
|
4884
5756
|
baseModelRevisionSchema,
|
|
4885
5757
|
behaviorSpecExampleSchema,
|
|
@@ -4917,22 +5789,6 @@ export {
|
|
|
4917
5789
|
examplesFromSpec,
|
|
4918
5790
|
examplesFromChatJsonl,
|
|
4919
5791
|
evaluationSuiteFromChatJsonl,
|
|
4920
|
-
reportInBackground,
|
|
4921
|
-
sanitizeLogLine,
|
|
4922
|
-
forwardStreamLines,
|
|
4923
|
-
localRuntimePackageRoot,
|
|
4924
|
-
buildBundledPythonCommand,
|
|
4925
|
-
withBundledPythonEnvironment,
|
|
4926
|
-
buildFoundationPythonCommand,
|
|
4927
|
-
withFoundationPythonEnvironment,
|
|
4928
|
-
runLoggedProcess,
|
|
4929
|
-
isTerminalRunState,
|
|
4930
|
-
defaultLocalHome,
|
|
4931
|
-
localStorePaths,
|
|
4932
|
-
createLocalStore,
|
|
4933
|
-
minimalMachineLearningEnvironment,
|
|
4934
|
-
withHuggingFaceCacheEnvironment,
|
|
4935
|
-
withOfflineHuggingFaceCacheEnvironment,
|
|
4936
5792
|
tokenF1,
|
|
4937
5793
|
verifyModelPrefetchCacheReport,
|
|
4938
5794
|
buildModelPrefetchPayload,
|
|
@@ -4949,4 +5805,4 @@ export {
|
|
|
4949
5805
|
runLocalPipeline,
|
|
4950
5806
|
runLocalFineTune
|
|
4951
5807
|
};
|
|
4952
|
-
//# sourceMappingURL=chunk-
|
|
5808
|
+
//# sourceMappingURL=chunk-AVMRHDFY.js.map
|