@tuned-tensor/cli 0.13.1-beta.2 → 0.13.1-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -2
- package/dist/{chunk-2P2RL23O.js → chunk-7VIUHQBI.js} +994 -845
- package/dist/chunk-7VIUHQBI.js.map +1 -0
- package/dist/index.js +257 -49
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +32 -9
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/architecture.md +4 -4
- package/examples/local-runtime/dry-runner.json +2 -2
- package/examples/local-runtime/local-runner.json +2 -2
- package/package.json +5 -5
- package/dist/chunk-2P2RL23O.js.map +0 -1
- /package/training/{local-runner → adapter}/pyproject.toml +0 -0
- /package/training/{local-runner → adapter}/src/evaluate.py +0 -0
- /package/training/{local-runner → adapter}/src/model_contract.py +0 -0
- /package/training/{local-runner → adapter}/src/prefetch.py +0 -0
- /package/training/{local-runner → adapter}/src/serve.py +0 -0
- /package/training/{local-runner → adapter}/src/sft_data.py +0 -0
- /package/training/{local-runner → adapter}/src/train.py +0 -0
- /package/training/{local-runner → adapter}/uv.lock +0 -0
|
@@ -834,717 +834,183 @@ async function assertUsableModelArtifact(uri) {
|
|
|
834
834
|
return inspection;
|
|
835
835
|
}
|
|
836
836
|
|
|
837
|
-
// src/local-runtime/
|
|
838
|
-
import {
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
var behaviorSpecExampleSchema = z.object({
|
|
842
|
-
input: z.string().min(1),
|
|
843
|
-
output: z.string().min(1)
|
|
844
|
-
}).strict();
|
|
845
|
-
var specSnapshotSchema = z.object({
|
|
846
|
-
name: z.string().min(1),
|
|
847
|
-
description: z.string().default(""),
|
|
848
|
-
system_prompt: z.string().default(""),
|
|
849
|
-
guidelines: z.array(z.string()).default([]),
|
|
850
|
-
examples: z.array(behaviorSpecExampleSchema).default([]),
|
|
851
|
-
constraints: z.array(z.string()).default([]),
|
|
852
|
-
base_model: z.string().transform((value) => canonicalizeTrainingModel(value))
|
|
853
|
-
}).strict();
|
|
854
|
-
var foundationHyperparametersSchema = z.object({
|
|
855
|
-
vocab_size: z.number().int().min(64).max(65536).default(256),
|
|
856
|
-
max_chars: z.number().int().min(1e3).max(2e9).default(2e4),
|
|
857
|
-
depth: z.number().int().min(1).max(64).default(2),
|
|
858
|
-
pretrain_steps: z.number().int().min(1).max(1e7).default(2),
|
|
859
|
-
finetune_steps: z.number().int().min(1).max(1e7).default(2),
|
|
860
|
-
rl_steps: z.number().int().min(0).max(1e7).default(0),
|
|
861
|
-
batch_size: z.number().int().min(1).max(1024).default(2),
|
|
862
|
-
sequence_length: z.number().int().min(16).max(8192).default(64),
|
|
863
|
-
nproc_per_node: z.number().int().min(1).max(16).default(1)
|
|
864
|
-
}).strict();
|
|
865
|
-
var localFoundationSpecFileSchema = z.object({
|
|
866
|
-
engine: z.literal("foundation"),
|
|
867
|
-
id: z.string().uuid().optional(),
|
|
868
|
-
name: z.string().min(1),
|
|
869
|
-
description: z.string().default(""),
|
|
870
|
-
system_prompt: z.string().default(""),
|
|
871
|
-
guidelines: z.array(z.string()).default([]),
|
|
872
|
-
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
873
|
-
constraints: z.array(z.string()).default([]),
|
|
874
|
-
foundation: foundationHyperparametersSchema
|
|
875
|
-
}).strict();
|
|
876
|
-
var datasetPrebuiltSchema = z.object({
|
|
877
|
-
training: z.string().min(1),
|
|
878
|
-
validation: z.string().min(1).optional(),
|
|
879
|
-
test: z.string().min(1).optional(),
|
|
880
|
-
format: datasetFormatSchema.default("chat_jsonl")
|
|
881
|
-
}).strict().superRefine((dataset, context) => {
|
|
882
|
-
if (!dataset.validation && !dataset.test) {
|
|
883
|
-
context.addIssue({
|
|
884
|
-
code: z.ZodIssueCode.custom,
|
|
885
|
-
path: ["validation"],
|
|
886
|
-
message: "Provide a distinct validation or test split for held-out evaluation"
|
|
887
|
-
});
|
|
888
|
-
}
|
|
889
|
-
});
|
|
890
|
-
var fineTuneHyperparametersSchema = z.object({
|
|
891
|
-
n_epochs: z.number().int().min(1).max(20).default(1),
|
|
892
|
-
learning_rate: z.number().positive().optional(),
|
|
893
|
-
batch_size: z.number().int().min(1).optional(),
|
|
894
|
-
lora_rank: z.number().int().min(1).optional(),
|
|
895
|
-
lora_alpha: z.number().int().min(1).optional(),
|
|
896
|
-
lora_dropout: z.number().min(0).max(1).optional(),
|
|
897
|
-
max_seq_length: z.number().int().min(128).max(32768).optional(),
|
|
898
|
-
gradient_accumulation_steps: z.number().int().min(1).optional(),
|
|
899
|
-
max_eval_examples: z.number().int().min(1).optional(),
|
|
900
|
-
/** Immutable Hugging Face commit used by every stage of the run. */
|
|
901
|
-
base_model_revision: baseModelRevisionSchema.optional()
|
|
902
|
-
}).strict();
|
|
903
|
-
var fineTuneRunRequestSchema = z.object({
|
|
904
|
-
run_id: z.string().uuid(),
|
|
905
|
-
user_id: z.string().min(1),
|
|
906
|
-
behavior_spec_id: z.string().uuid(),
|
|
907
|
-
run_number: z.number().int().min(1),
|
|
908
|
-
spec_snapshot: specSnapshotSchema,
|
|
909
|
-
hyperparameters: fineTuneHyperparametersSchema.default({ n_epochs: 1 }),
|
|
910
|
-
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
911
|
-
}).strict().superRefine((request, context) => {
|
|
912
|
-
if (request.spec_snapshot.examples.length === 0 && !request.dataset_prebuilt) {
|
|
913
|
-
context.addIssue({
|
|
914
|
-
code: z.ZodIssueCode.custom,
|
|
915
|
-
path: ["spec_snapshot", "examples"],
|
|
916
|
-
message: "Add examples or provide dataset_prebuilt training and validation files"
|
|
917
|
-
});
|
|
918
|
-
}
|
|
919
|
-
});
|
|
920
|
-
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
921
|
-
engine: z.literal("adapter").optional(),
|
|
922
|
-
id: z.string().uuid().optional(),
|
|
923
|
-
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
924
|
-
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
925
|
-
}).strict();
|
|
926
|
-
var localBehaviorSpecFileSchema = z.union([
|
|
927
|
-
localFoundationSpecFileSchema,
|
|
928
|
-
localAdapterSpecFileSchema
|
|
929
|
-
]);
|
|
930
|
-
var evalExampleResultSchema = z.object({
|
|
931
|
-
prompt: z.string(),
|
|
932
|
-
expected: z.string(),
|
|
933
|
-
actual: z.string(),
|
|
934
|
-
passed: z.boolean(),
|
|
935
|
-
score: z.number().min(0).max(1),
|
|
936
|
-
reasoning: z.string().nullable(),
|
|
937
|
-
latency_ms: z.number().int().nonnegative(),
|
|
938
|
-
scored_by: z.enum(["exact_match", "json_fields", "heuristic"]).optional()
|
|
939
|
-
}).strict();
|
|
940
|
-
var jsonFieldMetricsSchema = z.object({
|
|
941
|
-
fields: z.array(z.string()),
|
|
942
|
-
valid_json_count: z.number().int().nonnegative(),
|
|
943
|
-
valid_json_rate: z.number().min(0).max(1),
|
|
944
|
-
schema_match_count: z.number().int().nonnegative(),
|
|
945
|
-
schema_match_rate: z.number().min(0).max(1),
|
|
946
|
-
all_fields_match_count: z.number().int().nonnegative(),
|
|
947
|
-
all_fields_match_rate: z.number().min(0).max(1),
|
|
948
|
-
field_accuracy: z.record(z.string(), z.object({
|
|
949
|
-
correct: z.number().int().nonnegative(),
|
|
950
|
-
total: z.number().int().nonnegative(),
|
|
951
|
-
accuracy: z.number().min(0).max(1)
|
|
952
|
-
}).strict())
|
|
953
|
-
}).strict();
|
|
954
|
-
var evalSplitSchema = z.enum([
|
|
955
|
-
"spec_examples",
|
|
956
|
-
"spec_holdout",
|
|
957
|
-
"prebuilt_test",
|
|
958
|
-
"prebuilt_validation",
|
|
959
|
-
"general_regression"
|
|
960
|
-
]);
|
|
961
|
-
var evalReportSchema = z.object({
|
|
962
|
-
kind: z.enum(["baseline", "candidate"]),
|
|
963
|
-
model_id: z.string(),
|
|
964
|
-
total: z.number().int().nonnegative(),
|
|
965
|
-
eval_examples_total: z.number().int().nonnegative(),
|
|
966
|
-
eval_examples_used: z.number().int().nonnegative(),
|
|
967
|
-
eval_truncated: z.boolean(),
|
|
968
|
-
eval_split: evalSplitSchema.optional(),
|
|
969
|
-
eval_sample_seed: z.number().int().nullable().optional(),
|
|
970
|
-
avg_score: z.number().min(0).max(1),
|
|
971
|
-
pass_rate: z.number().min(0).max(1),
|
|
972
|
-
exact_match_rate: z.number().min(0).max(1),
|
|
973
|
-
avg_token_f1: z.number().min(0).max(1).optional(),
|
|
974
|
-
avg_latency_ms: z.number().int().nonnegative(),
|
|
975
|
-
cached: z.boolean().optional(),
|
|
976
|
-
cache_key: z.string().optional(),
|
|
977
|
-
results: z.array(evalExampleResultSchema),
|
|
978
|
-
artifact_uri: z.string(),
|
|
979
|
-
scoring_method: z.enum(["heuristic", "exact_match", "json_fields"]),
|
|
980
|
-
inference_provider: z.enum(["none", "transformers"]).optional(),
|
|
981
|
-
scoring_mode: z.enum(["exact_match", "json_fields"]).optional(),
|
|
982
|
-
json_field_metrics: jsonFieldMetricsSchema.optional(),
|
|
983
|
-
generation_config: z.record(z.string(), z.unknown()).optional(),
|
|
984
|
-
log_uri: z.string().optional()
|
|
985
|
-
}).strict();
|
|
986
|
-
var comparisonReportSchema = z.object({
|
|
987
|
-
avg_score_delta: z.number(),
|
|
988
|
-
pass_rate_delta: z.number(),
|
|
989
|
-
exact_match_rate_delta: z.number(),
|
|
990
|
-
token_f1_delta: z.number().optional(),
|
|
991
|
-
regressions: z.number().int().nonnegative(),
|
|
992
|
-
improvements: z.number().int().nonnegative(),
|
|
993
|
-
regressed_examples: z.array(z.object({
|
|
994
|
-
prompt: z.string(),
|
|
995
|
-
old_score: z.number(),
|
|
996
|
-
new_score: z.number()
|
|
997
|
-
}).strict())
|
|
998
|
-
}).strict();
|
|
999
|
-
var trainingReportSchema = z.object({
|
|
1000
|
-
provider: z.literal("local-uv"),
|
|
1001
|
-
training_job_name: z.string(),
|
|
1002
|
-
model_artifact_uri: z.string().optional(),
|
|
1003
|
-
base_model_artifact_uri: z.string().optional(),
|
|
1004
|
-
metrics: z.record(z.string(), z.unknown()).nullable(),
|
|
1005
|
-
exit_code: z.number().int().nullable(),
|
|
1006
|
-
log_uri: z.string(),
|
|
1007
|
-
command: z.array(z.string()).optional()
|
|
1008
|
-
}).strict();
|
|
1009
|
-
var runReportSchema = z.object({
|
|
1010
|
-
run_id: z.string().uuid(),
|
|
1011
|
-
behavior_spec_id: z.string().uuid(),
|
|
1012
|
-
user_id: z.string(),
|
|
1013
|
-
run_number: z.number().int().min(1),
|
|
1014
|
-
base_model: z.string(),
|
|
1015
|
-
fine_tuned_model_id: z.string(),
|
|
1016
|
-
status: z.enum(["completed", "failed"]),
|
|
1017
|
-
baseline: evalReportSchema,
|
|
1018
|
-
candidate: evalReportSchema,
|
|
1019
|
-
comparison: comparisonReportSchema,
|
|
1020
|
-
general_regression: z.object({
|
|
1021
|
-
dataset_uri: z.string(),
|
|
1022
|
-
dataset_sha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
1023
|
-
baseline: evalReportSchema,
|
|
1024
|
-
candidate: evalReportSchema,
|
|
1025
|
-
comparison: comparisonReportSchema,
|
|
1026
|
-
policy: z.object({
|
|
1027
|
-
max_score_drop: z.number().min(0).max(1),
|
|
1028
|
-
max_pass_rate_drop: z.number().min(0).max(1)
|
|
1029
|
-
}).strict(),
|
|
1030
|
-
passed: z.boolean(),
|
|
1031
|
-
failures: z.array(z.string())
|
|
1032
|
-
}).strict().optional(),
|
|
1033
|
-
training: trainingReportSchema,
|
|
1034
|
-
artifact_uris: z.object({
|
|
1035
|
-
dataset: z.string(),
|
|
1036
|
-
baseline_eval: z.string(),
|
|
1037
|
-
candidate_eval: z.string(),
|
|
1038
|
-
general_baseline_eval: z.string().optional(),
|
|
1039
|
-
general_candidate_eval: z.string().optional(),
|
|
1040
|
-
report: z.string()
|
|
1041
|
-
}).strict(),
|
|
1042
|
-
run_metadata: z.object({
|
|
1043
|
-
base_model: z.string(),
|
|
1044
|
-
fine_tuned_model_id: z.string(),
|
|
1045
|
-
dataset_prebuilt: z.boolean(),
|
|
1046
|
-
dataset_format: datasetFormatSchema.nullable().optional(),
|
|
1047
|
-
dataset_uri: z.string(),
|
|
1048
|
-
spec_example_count: z.number().int().nonnegative(),
|
|
1049
|
-
training_example_count: z.number().int().nonnegative().nullable(),
|
|
1050
|
-
eval_examples_total: z.number().int().nonnegative(),
|
|
1051
|
-
eval_examples_used: z.number().int().nonnegative(),
|
|
1052
|
-
eval_split: evalSplitSchema.optional(),
|
|
1053
|
-
eval_sample_seed: z.number().int().nullable().optional(),
|
|
1054
|
-
started_at: z.string(),
|
|
1055
|
-
completed_at: z.string(),
|
|
1056
|
-
elapsed_ms: z.number().int().nonnegative(),
|
|
1057
|
-
elapsed_seconds: z.number().nonnegative()
|
|
1058
|
-
}).strict(),
|
|
1059
|
-
created_at: z.string()
|
|
1060
|
-
}).strict();
|
|
1061
|
-
var inferenceConfigSchema = z.object({
|
|
1062
|
-
maxNewTokens: z.number().int().min(1).default(256),
|
|
1063
|
-
temperature: z.number().min(0).default(0),
|
|
1064
|
-
topP: z.number().min(0).max(1).default(1),
|
|
1065
|
-
device: z.enum(["cuda", "cpu"]).default("cuda")
|
|
1066
|
-
}).strict();
|
|
1067
|
-
var scoringConfigSchema = z.object({
|
|
1068
|
-
mode: z.enum(["exact_match", "json_fields"]).default("exact_match"),
|
|
1069
|
-
fields: z.array(z.string().min(1)).optional()
|
|
1070
|
-
}).strict();
|
|
1071
|
-
var evaluationConfigSchema = z.object({
|
|
1072
|
-
inference: inferenceConfigSchema.default({
|
|
1073
|
-
maxNewTokens: 256,
|
|
1074
|
-
temperature: 0,
|
|
1075
|
-
topP: 1,
|
|
1076
|
-
device: "cuda"
|
|
1077
|
-
}),
|
|
1078
|
-
scoring: scoringConfigSchema.default({ mode: "exact_match" }),
|
|
1079
|
-
timeoutMs: z.number().int().min(100).default(18e5),
|
|
1080
|
-
maxExamples: z.number().int().min(1).optional(),
|
|
1081
|
-
sampleSeed: z.number().int().optional(),
|
|
1082
|
-
baselineCache: z.boolean().default(true),
|
|
1083
|
-
generalRegression: z.object({
|
|
1084
|
-
dataset: z.string().min(1),
|
|
1085
|
-
systemPrompt: z.string().optional(),
|
|
1086
|
-
maxScoreDrop: z.number().min(0).max(1).default(0.03),
|
|
1087
|
-
maxPassRateDrop: z.number().min(0).max(1).default(0.05)
|
|
1088
|
-
}).strict().optional()
|
|
1089
|
-
}).strict();
|
|
1090
|
-
var localRunnerConfigSchema = z.object({
|
|
1091
|
-
storeRoot: z.string().optional(),
|
|
1092
|
-
artifactRoot: z.string().default(".tt-local/artifacts"),
|
|
1093
|
-
dryRun: z.boolean().default(false),
|
|
1094
|
-
paths: z.object({
|
|
1095
|
-
baseModel: z.string().optional(),
|
|
1096
|
-
modelCache: z.string().optional()
|
|
1097
|
-
}).strict().default({}),
|
|
1098
|
-
evaluation: evaluationConfigSchema.default({
|
|
1099
|
-
inference: {
|
|
1100
|
-
maxNewTokens: 256,
|
|
1101
|
-
temperature: 0,
|
|
1102
|
-
topP: 1,
|
|
1103
|
-
device: "cuda"
|
|
1104
|
-
},
|
|
1105
|
-
scoring: { mode: "exact_match" },
|
|
1106
|
-
timeoutMs: 18e5,
|
|
1107
|
-
baselineCache: true
|
|
1108
|
-
})
|
|
1109
|
-
}).strict();
|
|
1110
|
-
function isFoundationSpecFile(spec) {
|
|
1111
|
-
return spec.engine === "foundation";
|
|
1112
|
-
}
|
|
1113
|
-
|
|
1114
|
-
// src/local-runtime/local-project.ts
|
|
1115
|
-
import { createHash as createHash2, randomUUID as randomUUID2 } from "crypto";
|
|
1116
|
-
import { readFile as readFile2, stat as stat2, writeFile as writeFile2 } from "fs/promises";
|
|
1117
|
-
import { dirname as dirname2, isAbsolute as isAbsolute2, resolve as resolve3 } from "path";
|
|
1118
|
-
import { homedir } from "os";
|
|
1119
|
-
import { mkdir as mkdir2 } from "fs/promises";
|
|
1120
|
-
var DEFAULT_LOCAL_SPEC_PATH = "tunedtensor.json";
|
|
1121
|
-
function resolveLocalReference(value, baseDirectory) {
|
|
1122
|
-
if (typeof value !== "string" || !value || isAbsolute2(value) || /^[a-z][a-z0-9+.-]*:/i.test(value)) {
|
|
1123
|
-
return value;
|
|
1124
|
-
}
|
|
1125
|
-
if (value === "~") return homedir();
|
|
1126
|
-
if (value.startsWith("~/")) return resolve3(homedir(), value.slice(2));
|
|
1127
|
-
return resolve3(baseDirectory, value);
|
|
1128
|
-
}
|
|
1129
|
-
function resolveLocalRunInputPaths(raw, inputPath) {
|
|
1130
|
-
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw;
|
|
1131
|
-
const baseDirectory = dirname2(resolve3(inputPath));
|
|
1132
|
-
const value = structuredClone(raw);
|
|
1133
|
-
const dataset = value.dataset_prebuilt;
|
|
1134
|
-
if (dataset && typeof dataset === "object" && !Array.isArray(dataset)) {
|
|
1135
|
-
const fields = dataset;
|
|
1136
|
-
for (const key of ["training", "validation", "test"]) {
|
|
1137
|
-
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
1138
|
-
}
|
|
1139
|
-
}
|
|
1140
|
-
return value;
|
|
1141
|
-
}
|
|
1142
|
-
async function exists(path) {
|
|
1143
|
-
try {
|
|
1144
|
-
await stat2(path);
|
|
1145
|
-
return true;
|
|
1146
|
-
} catch {
|
|
1147
|
-
return false;
|
|
1148
|
-
}
|
|
1149
|
-
}
|
|
1150
|
-
async function initLocalSpecFile(args) {
|
|
1151
|
-
if (!args.force && await exists(args.outputPath)) {
|
|
1152
|
-
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
1153
|
-
}
|
|
1154
|
-
if (args.engine === "foundation") {
|
|
1155
|
-
const spec2 = localFoundationSpecFileSchema.parse({
|
|
1156
|
-
engine: "foundation",
|
|
1157
|
-
id: randomUUID2(),
|
|
1158
|
-
name: args.name,
|
|
1159
|
-
description: "From-scratch tokenizer, pretrain, chat SFT, and eval.",
|
|
1160
|
-
system_prompt: "You are a helpful assistant.",
|
|
1161
|
-
guidelines: [
|
|
1162
|
-
"Answer the user request directly."
|
|
1163
|
-
],
|
|
1164
|
-
constraints: [],
|
|
1165
|
-
examples: [
|
|
1166
|
-
{
|
|
1167
|
-
input: "Replace this with representative chat input.",
|
|
1168
|
-
output: "Replace this with the expected assistant reply."
|
|
1169
|
-
},
|
|
1170
|
-
{
|
|
1171
|
-
input: "Replace this with a different chat input.",
|
|
1172
|
-
output: "Replace this with the expected assistant reply."
|
|
1173
|
-
}
|
|
1174
|
-
],
|
|
1175
|
-
foundation: {
|
|
1176
|
-
depth: 2,
|
|
1177
|
-
pretrain_steps: 2,
|
|
1178
|
-
finetune_steps: 2,
|
|
1179
|
-
rl_steps: 0,
|
|
1180
|
-
vocab_size: 256,
|
|
1181
|
-
max_chars: 2e4,
|
|
1182
|
-
sequence_length: 64,
|
|
1183
|
-
batch_size: 2,
|
|
1184
|
-
nproc_per_node: 1
|
|
1185
|
-
}
|
|
1186
|
-
});
|
|
1187
|
-
await mkdir2(dirname2(args.outputPath), { recursive: true });
|
|
1188
|
-
await writeFile2(args.outputPath, `${JSON.stringify(spec2, null, 2)}
|
|
1189
|
-
`, "utf8");
|
|
1190
|
-
return spec2;
|
|
1191
|
-
}
|
|
1192
|
-
if (!args.baseModel) {
|
|
1193
|
-
throw new Error("Adapter specs require --model.");
|
|
1194
|
-
}
|
|
1195
|
-
const spec = localAdapterSpecFileSchema.parse({
|
|
1196
|
-
id: randomUUID2(),
|
|
1197
|
-
name: args.name,
|
|
1198
|
-
description: "",
|
|
1199
|
-
system_prompt: "Describe the behavior this local model should learn.",
|
|
1200
|
-
guidelines: [
|
|
1201
|
-
"Return concise, task-specific answers."
|
|
1202
|
-
],
|
|
1203
|
-
constraints: [],
|
|
1204
|
-
base_model: args.baseModel,
|
|
1205
|
-
examples: [
|
|
1206
|
-
{
|
|
1207
|
-
input: "Replace this with representative training input.",
|
|
1208
|
-
output: "Replace this with the expected training output."
|
|
1209
|
-
},
|
|
1210
|
-
{
|
|
1211
|
-
input: "Replace this with a different validation input.",
|
|
1212
|
-
output: "Replace this with the expected validation output."
|
|
1213
|
-
}
|
|
1214
|
-
],
|
|
1215
|
-
hyperparameters: {
|
|
1216
|
-
n_epochs: 1
|
|
1217
|
-
}
|
|
1218
|
-
});
|
|
1219
|
-
await mkdir2(dirname2(args.outputPath), { recursive: true });
|
|
1220
|
-
await writeFile2(args.outputPath, `${JSON.stringify(spec, null, 2)}
|
|
1221
|
-
`, "utf8");
|
|
1222
|
-
return spec;
|
|
1223
|
-
}
|
|
1224
|
-
async function initLocalRunnerConfigFile(args) {
|
|
1225
|
-
if (!args.force && await exists(args.outputPath)) {
|
|
1226
|
-
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
1227
|
-
}
|
|
1228
|
-
const config = {
|
|
1229
|
-
artifactRoot: ".tt-local/artifacts",
|
|
1230
|
-
storeRoot: ".tt-local/store",
|
|
1231
|
-
evaluation: {
|
|
1232
|
-
inference: {
|
|
1233
|
-
device: "cuda"
|
|
1234
|
-
},
|
|
1235
|
-
scoring: {
|
|
1236
|
-
mode: "exact_match"
|
|
1237
|
-
},
|
|
1238
|
-
timeoutMs: 18e5
|
|
1239
|
-
}
|
|
1240
|
-
};
|
|
1241
|
-
await mkdir2(dirname2(args.outputPath), { recursive: true });
|
|
1242
|
-
await writeFile2(args.outputPath, `${JSON.stringify(config, null, 2)}
|
|
1243
|
-
`, "utf8");
|
|
1244
|
-
return config;
|
|
1245
|
-
}
|
|
1246
|
-
function placeholderIssues(systemPrompt, examples) {
|
|
1247
|
-
const issues = [];
|
|
1248
|
-
if (/describe the behavior this local model should learn/i.test(systemPrompt)) {
|
|
1249
|
-
issues.push("system_prompt still contains the generated placeholder");
|
|
1250
|
-
}
|
|
1251
|
-
examples.forEach((example, index) => {
|
|
1252
|
-
if (/replace this with/i.test(example.input) || /replace this with/i.test(example.output)) {
|
|
1253
|
-
issues.push(`examples[${index}] still contains generated placeholder text`);
|
|
1254
|
-
}
|
|
1255
|
-
});
|
|
1256
|
-
return issues;
|
|
1257
|
-
}
|
|
1258
|
-
function generatedPlaceholderIssues(request) {
|
|
1259
|
-
return placeholderIssues(request.spec_snapshot.system_prompt, request.spec_snapshot.examples);
|
|
1260
|
-
}
|
|
1261
|
-
function assertLocalRunInputReady(request) {
|
|
1262
|
-
const issues = generatedPlaceholderIssues(request);
|
|
1263
|
-
if (issues.length > 0) {
|
|
1264
|
-
throw new Error(`Edit the generated behavior spec before training: ${issues.join("; ")}.`);
|
|
1265
|
-
}
|
|
1266
|
-
}
|
|
1267
|
-
function foundationPlaceholderIssues(spec) {
|
|
1268
|
-
return placeholderIssues(spec.system_prompt, spec.examples);
|
|
1269
|
-
}
|
|
1270
|
-
function assertFoundationSpecReady(spec) {
|
|
1271
|
-
const issues = foundationPlaceholderIssues(spec);
|
|
1272
|
-
if (issues.length > 0) {
|
|
1273
|
-
throw new Error(`Edit the generated foundation spec before training: ${issues.join("; ")}.`);
|
|
1274
|
-
}
|
|
1275
|
-
}
|
|
1276
|
-
function runRequestFromLocalSpec(spec, options = {}) {
|
|
1277
|
-
if (isFoundationSpecFile(spec)) {
|
|
1278
|
-
throw new Error(
|
|
1279
|
-
"Foundation specs cannot be converted into a LoRA run request. Use `tt pipeline run --spec`."
|
|
1280
|
-
);
|
|
1281
|
-
}
|
|
1282
|
-
const {
|
|
1283
|
-
id,
|
|
1284
|
-
engine: _engine,
|
|
1285
|
-
hyperparameters,
|
|
1286
|
-
dataset_prebuilt: datasetPrebuilt,
|
|
1287
|
-
...specSnapshot
|
|
1288
|
-
} = spec;
|
|
1289
|
-
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
1290
|
-
return fineTuneRunRequestSchema.parse({
|
|
1291
|
-
run_id: options.runId ?? randomUUID2(),
|
|
1292
|
-
user_id: "local-user",
|
|
1293
|
-
behavior_spec_id: behaviorSpecId,
|
|
1294
|
-
run_number: 1,
|
|
1295
|
-
spec_snapshot: specSnapshot,
|
|
1296
|
-
hyperparameters,
|
|
1297
|
-
dataset_prebuilt: datasetPrebuilt
|
|
1298
|
-
});
|
|
1299
|
-
}
|
|
1300
|
-
function deterministicBehaviorSpecId(spec) {
|
|
1301
|
-
const namespace = Buffer.from("9f783f01340457bea061942004259253", "hex");
|
|
1302
|
-
const digest = createHash2("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
|
|
1303
|
-
digest[6] = digest[6] & 15 | 80;
|
|
1304
|
-
digest[8] = digest[8] & 63 | 128;
|
|
1305
|
-
const hex = digest.toString("hex");
|
|
1306
|
-
return [
|
|
1307
|
-
hex.slice(0, 8),
|
|
1308
|
-
hex.slice(8, 12),
|
|
1309
|
-
hex.slice(12, 16),
|
|
1310
|
-
hex.slice(16, 20),
|
|
1311
|
-
hex.slice(20)
|
|
1312
|
-
].join("-");
|
|
1313
|
-
}
|
|
1314
|
-
async function loadLocalRunInput(path, options = {}) {
|
|
1315
|
-
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile2(path, "utf8")), path);
|
|
1316
|
-
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
1317
|
-
if (request.success) {
|
|
1318
|
-
return { kind: "request", path, request: request.data };
|
|
1319
|
-
}
|
|
1320
|
-
const spec = localBehaviorSpecFileSchema.parse(raw);
|
|
1321
|
-
if (isFoundationSpecFile(spec)) {
|
|
1322
|
-
return { kind: "foundation-spec", path, spec };
|
|
1323
|
-
}
|
|
1324
|
-
return {
|
|
1325
|
-
kind: "spec",
|
|
1326
|
-
path,
|
|
1327
|
-
spec,
|
|
1328
|
-
request: runRequestFromLocalSpec(spec, options)
|
|
1329
|
-
};
|
|
1330
|
-
}
|
|
1331
|
-
|
|
1332
|
-
// src/local-runtime/dataset.ts
|
|
1333
|
-
import { readFile as readFile3 } from "fs/promises";
|
|
1334
|
-
async function loadNormalizedChatJsonl(path) {
|
|
1335
|
-
const text = await readFile3(path, "utf8");
|
|
1336
|
-
const rows = [];
|
|
1337
|
-
const jsonLines = [];
|
|
1338
|
-
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
1339
|
-
if (!line.trim()) continue;
|
|
1340
|
-
let value;
|
|
1341
|
-
try {
|
|
1342
|
-
value = JSON.parse(line);
|
|
1343
|
-
} catch (error) {
|
|
1344
|
-
throw new Error(`Invalid chat JSONL row ${index + 1}: malformed JSON`, { cause: error });
|
|
1345
|
-
}
|
|
1346
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1347
|
-
throw new Error(`Invalid chat JSONL row ${index + 1}: expected an object`);
|
|
1348
|
-
}
|
|
1349
|
-
const messages = value.messages;
|
|
1350
|
-
if (!Array.isArray(messages) || ![2, 3].includes(messages.length) || messages.some(
|
|
1351
|
-
(message) => !message || typeof message !== "object" || Array.isArray(message) || !["system", "user", "assistant"].includes(String(message.role)) || typeof message.content !== "string"
|
|
1352
|
-
)) {
|
|
1353
|
-
throw new Error(
|
|
1354
|
-
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one user message, and one assistant answer`
|
|
1355
|
-
);
|
|
1356
|
-
}
|
|
1357
|
-
const row = { messages };
|
|
1358
|
-
const offset = row.messages.length === 3 ? 1 : 0;
|
|
1359
|
-
if (offset === 1 && row.messages[0]?.role !== "system" || row.messages[offset]?.role !== "user" || !row.messages[offset]?.content.trim() || row.messages[offset + 1]?.role !== "assistant" || !row.messages[offset + 1]?.content.trim()) {
|
|
1360
|
-
throw new Error(
|
|
1361
|
-
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one non-empty user message, and one non-empty assistant answer`
|
|
1362
|
-
);
|
|
1363
|
-
}
|
|
1364
|
-
rows.push(row);
|
|
1365
|
-
jsonLines.push(JSON.stringify(row));
|
|
1366
|
-
}
|
|
1367
|
-
if (rows.length === 0) throw new Error(`Chat JSONL contains no examples: ${path}`);
|
|
1368
|
-
return {
|
|
1369
|
-
rows,
|
|
1370
|
-
jsonl: jsonLines.join("\n")
|
|
1371
|
-
};
|
|
1372
|
-
}
|
|
1373
|
-
async function normalizeChatJsonlForRelocation(path) {
|
|
1374
|
-
return (await loadNormalizedChatJsonl(path)).jsonl;
|
|
1375
|
-
}
|
|
1376
|
-
function buildSystemMessage(spec) {
|
|
1377
|
-
const parts = [];
|
|
1378
|
-
if (spec.system_prompt.trim()) parts.push(spec.system_prompt.trim());
|
|
1379
|
-
if (spec.guidelines.length > 0) {
|
|
1380
|
-
parts.push(`Guidelines:
|
|
1381
|
-
${spec.guidelines.map((guideline) => `- ${guideline}`).join("\n")}`);
|
|
1382
|
-
}
|
|
1383
|
-
if (spec.constraints.length > 0) {
|
|
1384
|
-
parts.push(`Constraints:
|
|
1385
|
-
${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
|
|
1386
|
-
}
|
|
1387
|
-
return parts.join("\n\n") || "Follow the demonstrated behavior.";
|
|
1388
|
-
}
|
|
1389
|
-
function exampleToChatRow(spec, example) {
|
|
1390
|
-
const system = buildSystemMessage(spec);
|
|
1391
|
-
return {
|
|
1392
|
-
messages: [
|
|
1393
|
-
{ role: "system", content: system },
|
|
1394
|
-
{ role: "user", content: example.input },
|
|
1395
|
-
{ role: "assistant", content: example.output }
|
|
1396
|
-
]
|
|
1397
|
-
};
|
|
1398
|
-
}
|
|
1399
|
-
function compileSpecToJsonl(spec) {
|
|
1400
|
-
return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
|
|
1401
|
-
}
|
|
1402
|
-
function examplesFromSpec(spec) {
|
|
1403
|
-
return spec.examples;
|
|
1404
|
-
}
|
|
1405
|
-
async function examplesFromChatJsonl(path) {
|
|
1406
|
-
const normalized = await loadNormalizedChatJsonl(path);
|
|
1407
|
-
const examples = [];
|
|
1408
|
-
for (const [index, row] of normalized.rows.entries()) {
|
|
1409
|
-
const user = row.messages.at(-2);
|
|
1410
|
-
const assistant = row.messages.at(-1);
|
|
1411
|
-
examples.push({
|
|
1412
|
-
input: user.content,
|
|
1413
|
-
output: assistant.content
|
|
1414
|
-
});
|
|
1415
|
-
}
|
|
1416
|
-
return examples;
|
|
1417
|
-
}
|
|
1418
|
-
async function evaluationSuiteFromChatJsonl(path, configuredSystem) {
|
|
1419
|
-
const normalized = await loadNormalizedChatJsonl(path);
|
|
1420
|
-
const inputIdentities = /* @__PURE__ */ new Set();
|
|
1421
|
-
for (const row of normalized.rows) {
|
|
1422
|
-
const input = row.messages.at(-2).content;
|
|
1423
|
-
const identity = input.trim().replace(/\s+/g, " ").toLowerCase();
|
|
1424
|
-
if (inputIdentities.has(identity)) {
|
|
1425
|
-
throw new Error(
|
|
1426
|
-
"General regression data contains duplicate inputs; evaluation prompts must be unique."
|
|
1427
|
-
);
|
|
1428
|
-
}
|
|
1429
|
-
inputIdentities.add(identity);
|
|
1430
|
-
}
|
|
1431
|
-
const systems = new Set(
|
|
1432
|
-
normalized.rows.map((row) => row.messages.find((message) => message.role === "system")?.content).filter((value) => value !== void 0)
|
|
1433
|
-
);
|
|
1434
|
-
if (systems.size > 1) {
|
|
1435
|
-
throw new Error("General regression rows must use one consistent system message.");
|
|
1436
|
-
}
|
|
1437
|
-
const embeddedSystem = [...systems][0];
|
|
1438
|
-
if (configuredSystem !== void 0 && embeddedSystem !== void 0 && configuredSystem !== embeddedSystem) {
|
|
1439
|
-
throw new Error(
|
|
1440
|
-
"evaluation.generalRegression.systemPrompt must match the system message embedded in the dataset."
|
|
1441
|
-
);
|
|
1442
|
-
}
|
|
1443
|
-
return {
|
|
1444
|
-
examples: normalized.rows.map((row) => ({
|
|
1445
|
-
input: row.messages.at(-2).content,
|
|
1446
|
-
output: row.messages.at(-1).content
|
|
1447
|
-
})),
|
|
1448
|
-
system: configuredSystem ?? embeddedSystem ?? "You are a helpful assistant."
|
|
1449
|
-
};
|
|
1450
|
-
}
|
|
1451
|
-
|
|
1452
|
-
// src/local-runtime/run-reporter.ts
|
|
1453
|
-
function reportInBackground(action) {
|
|
1454
|
-
try {
|
|
1455
|
-
void Promise.resolve(action()).catch(() => void 0);
|
|
1456
|
-
} catch {
|
|
1457
|
-
}
|
|
1458
|
-
}
|
|
1459
|
-
function sanitizeLogLine(line) {
|
|
1460
|
-
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
1461
|
-
}
|
|
1462
|
-
function forwardStreamLines(stream, onLine) {
|
|
1463
|
-
let buffered = "";
|
|
1464
|
-
stream.setEncoding("utf8");
|
|
1465
|
-
stream.on("data", (chunk) => {
|
|
1466
|
-
buffered += chunk;
|
|
1467
|
-
const lines = buffered.split(/\r\n|\n|\r/);
|
|
1468
|
-
buffered = lines.pop() ?? "";
|
|
1469
|
-
for (const line of lines) {
|
|
1470
|
-
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
1471
|
-
}
|
|
1472
|
-
});
|
|
1473
|
-
stream.on("end", () => {
|
|
1474
|
-
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
1475
|
-
});
|
|
1476
|
-
}
|
|
837
|
+
// src/local-runtime/store.ts
|
|
838
|
+
import { appendFile, chmod, copyFile as copyFile2, lstat as lstat3, mkdir as mkdir2, readdir as readdir3, readFile as readFile2, rename as rename2, rm, stat as stat2, writeFile as writeFile2 } from "fs/promises";
|
|
839
|
+
import { dirname as dirname2, join as join3, resolve as resolve4, sep as sep2 } from "path";
|
|
840
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
1477
841
|
|
|
1478
|
-
// src/
|
|
842
|
+
// src/paths.ts
|
|
1479
843
|
import { existsSync } from "fs";
|
|
1480
|
-
import {
|
|
1481
|
-
import {
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
return
|
|
844
|
+
import { homedir } from "os";
|
|
845
|
+
import { isAbsolute as isAbsolute2, join as join2, resolve as resolve3 } from "path";
|
|
846
|
+
var TUNED_TENSOR_DIR = ".tuned-tensor";
|
|
847
|
+
var DEFAULT_ARTIFACT_ROOT = `${TUNED_TENSOR_DIR}/artifacts`;
|
|
848
|
+
var DEFAULT_PROJECT_STORE_ROOT = `${TUNED_TENSOR_DIR}/store`;
|
|
849
|
+
var DEFAULT_FOUNDATION_RUNS_DIR = `${TUNED_TENSOR_DIR}/foundation-runs`;
|
|
850
|
+
function envHome(env = process.env) {
|
|
851
|
+
const home = env.HOME?.trim();
|
|
852
|
+
return home ? resolve3(home) : homedir();
|
|
853
|
+
}
|
|
854
|
+
function expandUserPath(value, env = process.env, baseDirectory = process.cwd()) {
|
|
855
|
+
const trimmed = value.trim();
|
|
856
|
+
if (trimmed === "~") return envHome(env);
|
|
857
|
+
if (trimmed.startsWith("~/")) return resolve3(envHome(env), trimmed.slice(2));
|
|
858
|
+
return isAbsolute2(trimmed) ? resolve3(trimmed) : resolve3(baseDirectory, trimmed);
|
|
859
|
+
}
|
|
860
|
+
function getTunedTensorHome(env = process.env) {
|
|
861
|
+
const override = env.TUNED_TENSOR_HOME?.trim();
|
|
862
|
+
if (override) return expandUserPath(override, env);
|
|
863
|
+
return join2(envHome(env), TUNED_TENSOR_DIR);
|
|
864
|
+
}
|
|
865
|
+
function legacyConfigDir(env) {
|
|
866
|
+
const configHome = env.XDG_CONFIG_HOME?.trim() ? resolve3(env.XDG_CONFIG_HOME) : join2(envHome(env), ".config");
|
|
867
|
+
return join2(configHome, "tuned-tensor");
|
|
868
|
+
}
|
|
869
|
+
function getConfigDir(env = process.env) {
|
|
870
|
+
const next = getTunedTensorHome(env);
|
|
871
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
872
|
+
const legacy = legacyConfigDir(env);
|
|
873
|
+
if (!existsSync(join2(next, "config.json")) && existsSync(join2(legacy, "config.json"))) {
|
|
874
|
+
return legacy;
|
|
875
|
+
}
|
|
876
|
+
return next;
|
|
877
|
+
}
|
|
878
|
+
function getAgentConfigDir(env = process.env) {
|
|
879
|
+
const next = join2(getTunedTensorHome(env), "agent");
|
|
880
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
881
|
+
const legacy = join2(legacyConfigDir(env), "agent");
|
|
882
|
+
if (!existsSync(next) && existsSync(legacy)) return legacy;
|
|
883
|
+
return next;
|
|
884
|
+
}
|
|
885
|
+
function defaultStoreRoot(env = process.env) {
|
|
886
|
+
const override = env.TT_LOCAL_HOME?.trim();
|
|
887
|
+
if (override) return expandUserPath(override, env);
|
|
888
|
+
const next = join2(getTunedTensorHome(env), "store");
|
|
889
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
890
|
+
const legacy = join2(envHome(env), ".tuned-tensor-local");
|
|
891
|
+
if (!existsSync(next) && existsSync(legacy)) return legacy;
|
|
892
|
+
return next;
|
|
893
|
+
}
|
|
894
|
+
function defaultCacheRoot(env = process.env) {
|
|
895
|
+
return join2(getTunedTensorHome(env), "cache");
|
|
896
|
+
}
|
|
897
|
+
function pythonEnvironmentPath(kind, hash, env = process.env) {
|
|
898
|
+
const next = join2(defaultCacheRoot(env), kind, hash);
|
|
899
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
900
|
+
const cacheHome = env.XDG_CACHE_HOME?.trim() ? resolve3(env.XDG_CACHE_HOME) : join2(envHome(env), ".cache");
|
|
901
|
+
const legacy = join2(cacheHome, "tuned-tensor-local", kind, hash);
|
|
902
|
+
if (!existsSync(next) && existsSync(legacy)) return legacy;
|
|
903
|
+
return next;
|
|
1495
904
|
}
|
|
1496
905
|
|
|
1497
906
|
// src/local-runtime/store.ts
|
|
1498
|
-
import { appendFile, copyFile as copyFile2, mkdir as mkdir3, readdir as readdir3, readFile as readFile4, rename as rename2, rm, stat as stat3, writeFile as writeFile3 } from "fs/promises";
|
|
1499
|
-
import { homedir as homedir2 } from "os";
|
|
1500
|
-
import { dirname as dirname4, join as join2, resolve as resolve5, sep as sep2 } from "path";
|
|
1501
|
-
import { randomUUID as randomUUID3 } from "crypto";
|
|
1502
907
|
function isTerminalRunState(state) {
|
|
1503
908
|
return state.status === "stage_completed" || state.status === "completed" || state.status === "failed" || state.status === "cancelled" && state.current_stage === "cancelled" && Boolean(state.completed_at);
|
|
1504
909
|
}
|
|
1505
910
|
function defaultLocalHome() {
|
|
1506
|
-
return
|
|
911
|
+
return defaultStoreRoot();
|
|
1507
912
|
}
|
|
1508
913
|
function localStorePaths(root) {
|
|
1509
914
|
return {
|
|
1510
915
|
root,
|
|
1511
|
-
specsDir:
|
|
1512
|
-
runsDir:
|
|
1513
|
-
modelsDir:
|
|
1514
|
-
datasetsDir:
|
|
916
|
+
specsDir: join3(root, "specs"),
|
|
917
|
+
runsDir: join3(root, "runs"),
|
|
918
|
+
modelsDir: join3(root, "models"),
|
|
919
|
+
datasetsDir: join3(root, "datasets")
|
|
1515
920
|
};
|
|
1516
921
|
}
|
|
1517
|
-
async function
|
|
922
|
+
async function exists(path) {
|
|
1518
923
|
try {
|
|
1519
|
-
await
|
|
924
|
+
await stat2(path);
|
|
1520
925
|
return true;
|
|
1521
926
|
} catch {
|
|
1522
927
|
return false;
|
|
1523
928
|
}
|
|
1524
929
|
}
|
|
1525
|
-
async function
|
|
1526
|
-
await
|
|
1527
|
-
const
|
|
1528
|
-
|
|
1529
|
-
|
|
930
|
+
async function ensurePrivateDirectory(path) {
|
|
931
|
+
await mkdir2(path, { recursive: true, mode: 448 });
|
|
932
|
+
const metadata = await lstat3(path);
|
|
933
|
+
if (metadata.isSymbolicLink()) {
|
|
934
|
+
throw new Error(`Refusing symbolic link in local store: ${path}`);
|
|
935
|
+
}
|
|
936
|
+
if (!metadata.isDirectory()) {
|
|
937
|
+
throw new Error(`Expected local store directory: ${path}`);
|
|
938
|
+
}
|
|
939
|
+
await chmod(path, 448);
|
|
940
|
+
}
|
|
941
|
+
async function hardenPrivateTree(path) {
|
|
942
|
+
const metadata = await lstat3(path);
|
|
943
|
+
if (metadata.isSymbolicLink()) {
|
|
944
|
+
throw new Error(`Refusing symbolic link in local store: ${path}`);
|
|
945
|
+
}
|
|
946
|
+
if (metadata.isDirectory()) {
|
|
947
|
+
await chmod(path, 448);
|
|
948
|
+
for (const entry of await readdir3(path)) {
|
|
949
|
+
await hardenPrivateTree(join3(path, entry));
|
|
950
|
+
}
|
|
951
|
+
return;
|
|
952
|
+
}
|
|
953
|
+
if (metadata.isFile()) {
|
|
954
|
+
await chmod(path, 384);
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
957
|
+
throw new Error(`Refusing unsupported filesystem entry in local store: ${path}`);
|
|
958
|
+
}
|
|
959
|
+
async function hardenExistingLocalStore(root) {
|
|
960
|
+
const resolvedRoot = resolve4(root);
|
|
961
|
+
let metadata;
|
|
962
|
+
try {
|
|
963
|
+
metadata = await lstat3(resolvedRoot);
|
|
964
|
+
} catch (error) {
|
|
965
|
+
if (error.code === "ENOENT") return false;
|
|
966
|
+
throw error;
|
|
967
|
+
}
|
|
968
|
+
if (metadata.isSymbolicLink()) {
|
|
969
|
+
throw new Error(`Refusing symbolic link in local store: ${resolvedRoot}`);
|
|
970
|
+
}
|
|
971
|
+
if (!metadata.isDirectory()) {
|
|
972
|
+
throw new Error(`Expected local store directory: ${resolvedRoot}`);
|
|
973
|
+
}
|
|
974
|
+
await hardenPrivateTree(resolvedRoot);
|
|
975
|
+
return true;
|
|
976
|
+
}
|
|
977
|
+
async function writePrivateJsonAtomic(path, value) {
|
|
978
|
+
await ensurePrivateDirectory(dirname2(path));
|
|
979
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.${randomUUID2()}.tmp`;
|
|
980
|
+
await writeFile2(tmp, `${JSON.stringify(value, null, 2)}
|
|
981
|
+
`, {
|
|
982
|
+
encoding: "utf8",
|
|
983
|
+
mode: 384
|
|
984
|
+
});
|
|
1530
985
|
await rename2(tmp, path);
|
|
986
|
+
await chmod(path, 384);
|
|
1531
987
|
}
|
|
1532
988
|
async function readJson2(path) {
|
|
1533
|
-
return JSON.parse(await
|
|
989
|
+
return JSON.parse(await readFile2(path, "utf8"));
|
|
1534
990
|
}
|
|
1535
991
|
async function appendJsonl(path, value) {
|
|
1536
|
-
await
|
|
992
|
+
await ensurePrivateDirectory(dirname2(path));
|
|
1537
993
|
await appendFile(path, `${JSON.stringify(value)}
|
|
1538
|
-
`,
|
|
994
|
+
`, {
|
|
995
|
+
encoding: "utf8",
|
|
996
|
+
mode: 384
|
|
997
|
+
});
|
|
998
|
+
await chmod(path, 384);
|
|
999
|
+
}
|
|
1000
|
+
async function writePrivateText(path, value) {
|
|
1001
|
+
await ensurePrivateDirectory(dirname2(path));
|
|
1002
|
+
await writeFile2(path, value, { encoding: "utf8", mode: 384 });
|
|
1003
|
+
await chmod(path, 384);
|
|
1539
1004
|
}
|
|
1540
1005
|
async function readJsonl(path) {
|
|
1541
|
-
if (!await
|
|
1542
|
-
return (await
|
|
1006
|
+
if (!await exists(path)) return [];
|
|
1007
|
+
return (await readFile2(path, "utf8")).split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
|
|
1543
1008
|
}
|
|
1544
1009
|
async function copyIfExists(from, to) {
|
|
1545
|
-
if (!await
|
|
1546
|
-
await
|
|
1010
|
+
if (!await exists(from)) return;
|
|
1011
|
+
await ensurePrivateDirectory(dirname2(to));
|
|
1547
1012
|
await copyFile2(from, to);
|
|
1013
|
+
await chmod(to, 384);
|
|
1548
1014
|
}
|
|
1549
1015
|
async function childDirectoryNames(path) {
|
|
1550
1016
|
try {
|
|
@@ -1571,41 +1037,51 @@ function findByIdOrPrefix(records, id) {
|
|
|
1571
1037
|
return records.find((record) => record.id === id) ?? records.find((record) => record.id.startsWith(id));
|
|
1572
1038
|
}
|
|
1573
1039
|
function isPathInside(root, candidate) {
|
|
1574
|
-
const resolvedRoot =
|
|
1575
|
-
const resolvedCandidate =
|
|
1040
|
+
const resolvedRoot = resolve4(root);
|
|
1041
|
+
const resolvedCandidate = resolve4(candidate);
|
|
1576
1042
|
return resolvedCandidate === resolvedRoot || resolvedCandidate.startsWith(`${resolvedRoot}${sep2}`);
|
|
1577
1043
|
}
|
|
1578
1044
|
function createLocalStore(root = defaultLocalHome()) {
|
|
1579
|
-
const resolvedRoot =
|
|
1045
|
+
const resolvedRoot = resolve4(root);
|
|
1580
1046
|
const paths = localStorePaths(resolvedRoot);
|
|
1581
|
-
const runDir = (id) =>
|
|
1582
|
-
const runStatePath = (id) =>
|
|
1583
|
-
const runEventsPath = (id) =>
|
|
1584
|
-
const runRequestPath = (id) =>
|
|
1585
|
-
const runReportPath = (id) =>
|
|
1586
|
-
const cancellationPath = (id) =>
|
|
1587
|
-
const specDir = (id) =>
|
|
1588
|
-
const specPath = (id) =>
|
|
1589
|
-
const modelPath = (id) =>
|
|
1590
|
-
|
|
1047
|
+
const runDir = (id) => join3(paths.runsDir, id);
|
|
1048
|
+
const runStatePath = (id) => join3(runDir(id), "state.json");
|
|
1049
|
+
const runEventsPath = (id) => join3(runDir(id), "progress.jsonl");
|
|
1050
|
+
const runRequestPath = (id) => join3(runDir(id), "request.json");
|
|
1051
|
+
const runReportPath = (id) => join3(runDir(id), "run-report.json");
|
|
1052
|
+
const cancellationPath = (id) => join3(runDir(id), "cancel.requested");
|
|
1053
|
+
const specDir = (id) => join3(paths.specsDir, id);
|
|
1054
|
+
const specPath = (id) => join3(specDir(id), "spec.json");
|
|
1055
|
+
const modelPath = (id) => join3(paths.modelsDir, id, "model.json");
|
|
1056
|
+
let ensurePromise;
|
|
1057
|
+
async function ensureOnce() {
|
|
1058
|
+
await ensurePrivateDirectory(paths.root);
|
|
1591
1059
|
await Promise.all([
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1060
|
+
ensurePrivateDirectory(paths.specsDir),
|
|
1061
|
+
ensurePrivateDirectory(paths.runsDir),
|
|
1062
|
+
ensurePrivateDirectory(paths.modelsDir),
|
|
1063
|
+
ensurePrivateDirectory(paths.datasetsDir)
|
|
1596
1064
|
]);
|
|
1065
|
+
await hardenExistingLocalStore(paths.root);
|
|
1066
|
+
}
|
|
1067
|
+
async function ensure() {
|
|
1068
|
+
ensurePromise ??= ensureOnce().catch((error) => {
|
|
1069
|
+
ensurePromise = void 0;
|
|
1070
|
+
throw error;
|
|
1071
|
+
});
|
|
1072
|
+
await ensurePromise;
|
|
1597
1073
|
}
|
|
1598
1074
|
async function writeRunState(state) {
|
|
1599
|
-
await
|
|
1075
|
+
await writePrivateJsonAtomic(runStatePath(state.id), state);
|
|
1600
1076
|
return state;
|
|
1601
1077
|
}
|
|
1602
1078
|
async function readSpecRecord(id) {
|
|
1603
1079
|
const path = specPath(id);
|
|
1604
|
-
if (!await
|
|
1080
|
+
if (!await exists(path)) return null;
|
|
1605
1081
|
const [spec, fileStats, directoryStats] = await Promise.all([
|
|
1606
1082
|
readJson2(path),
|
|
1607
|
-
|
|
1608
|
-
|
|
1083
|
+
stat2(path),
|
|
1084
|
+
stat2(specDir(id))
|
|
1609
1085
|
]);
|
|
1610
1086
|
const createdAt = directoryStats.birthtimeMs > 0 ? directoryStats.birthtime : directoryStats.ctime;
|
|
1611
1087
|
return {
|
|
@@ -1631,7 +1107,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1631
1107
|
const records = [];
|
|
1632
1108
|
for (const id of await childDirectoryNames(paths.runsDir)) {
|
|
1633
1109
|
const path = runStatePath(id);
|
|
1634
|
-
if (await
|
|
1110
|
+
if (await exists(path)) records.push(await readJson2(path));
|
|
1635
1111
|
}
|
|
1636
1112
|
return records.sort(compareUpdatedAt);
|
|
1637
1113
|
}
|
|
@@ -1640,27 +1116,27 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1640
1116
|
const records = [];
|
|
1641
1117
|
for (const id of await childDirectoryNames(paths.modelsDir)) {
|
|
1642
1118
|
const path = modelPath(id);
|
|
1643
|
-
if (await
|
|
1119
|
+
if (await exists(path)) records.push(await readJson2(path));
|
|
1644
1120
|
}
|
|
1645
1121
|
return records.sort(compareCreatedAt);
|
|
1646
1122
|
}
|
|
1647
1123
|
async function importSpecRecord(specId, spec) {
|
|
1648
1124
|
await ensure();
|
|
1649
|
-
await
|
|
1650
|
-
await
|
|
1125
|
+
await ensurePrivateDirectory(specDir(specId));
|
|
1126
|
+
await writePrivateJsonAtomic(specPath(specId), spec);
|
|
1651
1127
|
const record = await readSpecRecord(specId);
|
|
1652
1128
|
if (!record) throw new Error(`Failed to persist spec: ${specId}`);
|
|
1653
1129
|
return record;
|
|
1654
1130
|
}
|
|
1655
1131
|
async function appendRunEvent(state, event) {
|
|
1656
1132
|
const row = {
|
|
1657
|
-
id:
|
|
1133
|
+
id: randomUUID2(),
|
|
1658
1134
|
run_id: state.id,
|
|
1659
1135
|
occurred_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1660
1136
|
...event
|
|
1661
1137
|
};
|
|
1662
1138
|
await appendJsonl(runEventsPath(state.id), row);
|
|
1663
|
-
await appendJsonl(
|
|
1139
|
+
await appendJsonl(join3(state.artifact_dir, "progress.jsonl"), {
|
|
1664
1140
|
at: row.occurred_at,
|
|
1665
1141
|
stage: row.stage,
|
|
1666
1142
|
status: row.status,
|
|
@@ -1690,6 +1166,10 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1690
1166
|
async function preserveCancellationRequestState(runId) {
|
|
1691
1167
|
const previous = await getRun(runId);
|
|
1692
1168
|
if (isTerminalRunState(previous)) return previous;
|
|
1169
|
+
await Promise.all([
|
|
1170
|
+
rm(join3(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true }),
|
|
1171
|
+
rm(runReportPath(previous.id), { force: true })
|
|
1172
|
+
]);
|
|
1693
1173
|
if (previous.status === "cancelled" && previous.current_stage === "cancel_requested") return previous;
|
|
1694
1174
|
const requested = {
|
|
1695
1175
|
...previous,
|
|
@@ -1704,19 +1184,20 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1704
1184
|
}
|
|
1705
1185
|
async function resolveRunId(id) {
|
|
1706
1186
|
await ensure();
|
|
1707
|
-
if (await
|
|
1187
|
+
if (await exists(runStatePath(id))) return id;
|
|
1708
1188
|
const record = findByIdOrPrefix(await listRunRecords(), id);
|
|
1709
1189
|
if (!record) throw new Error(`Run not found: ${id}`);
|
|
1710
1190
|
return record.id;
|
|
1711
1191
|
}
|
|
1712
1192
|
async function getRun(id) {
|
|
1193
|
+
await ensure();
|
|
1713
1194
|
return readJson2(runStatePath(await resolveRunId(id)));
|
|
1714
1195
|
}
|
|
1715
1196
|
async function registerModelRecord(args) {
|
|
1716
1197
|
if (!args.training.model_artifact_uri || args.training.metrics?.dry_run === true) return null;
|
|
1717
1198
|
const previous = await getRun(args.request.run_id);
|
|
1718
1199
|
const modelId = `local-${args.request.run_id}`;
|
|
1719
|
-
const existingModel = await
|
|
1200
|
+
const existingModel = await exists(modelPath(modelId)) ? await readJson2(modelPath(modelId)) : void 0;
|
|
1720
1201
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1721
1202
|
const model = {
|
|
1722
1203
|
id: modelId,
|
|
@@ -1730,7 +1211,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1730
1211
|
metrics: args.training.metrics,
|
|
1731
1212
|
created_at: existingModel?.created_at ?? args.createdAt ?? now
|
|
1732
1213
|
};
|
|
1733
|
-
await
|
|
1214
|
+
await writePrivateJsonAtomic(modelPath(model.id), model);
|
|
1734
1215
|
const alreadyRegistered = previous.model_id === modelId;
|
|
1735
1216
|
const state = {
|
|
1736
1217
|
...previous,
|
|
@@ -1780,7 +1261,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1780
1261
|
updated_at: now,
|
|
1781
1262
|
started_at: now
|
|
1782
1263
|
};
|
|
1783
|
-
await
|
|
1264
|
+
await writePrivateJsonAtomic(runRequestPath(request.run_id), request);
|
|
1784
1265
|
await writeRunState(state);
|
|
1785
1266
|
await appendRunEvent(state, { stage: "queued", status: "queued", message: "Run queued." });
|
|
1786
1267
|
return state;
|
|
@@ -1793,7 +1274,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1793
1274
|
);
|
|
1794
1275
|
}
|
|
1795
1276
|
await importSpecRecord(request.behavior_spec_id, request.spec_snapshot);
|
|
1796
|
-
await
|
|
1277
|
+
await writePrivateJsonAtomic(runRequestPath(request.run_id), request);
|
|
1797
1278
|
const state = {
|
|
1798
1279
|
...previous,
|
|
1799
1280
|
artifact_dir: artifactDir ?? previous.artifact_dir,
|
|
@@ -1806,7 +1287,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1806
1287
|
},
|
|
1807
1288
|
async updateRun({ runId, status, stage, message, details }) {
|
|
1808
1289
|
const previous = await getRun(runId);
|
|
1809
|
-
if (status !== "cancelled" && await
|
|
1290
|
+
if (status !== "cancelled" && await exists(cancellationPath(previous.id))) {
|
|
1810
1291
|
return preserveCancellationRequestState(previous.id);
|
|
1811
1292
|
}
|
|
1812
1293
|
const active = status !== "stage_completed" && status !== "completed" && status !== "failed" && status !== "cancelled";
|
|
@@ -1821,7 +1302,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1821
1302
|
updated_at: now
|
|
1822
1303
|
};
|
|
1823
1304
|
await writeRunState(state);
|
|
1824
|
-
if (status !== "cancelled" && await
|
|
1305
|
+
if (status !== "cancelled" && await exists(cancellationPath(previous.id))) {
|
|
1825
1306
|
return preserveCancellationRequestState(previous.id);
|
|
1826
1307
|
}
|
|
1827
1308
|
await appendRunEvent(state, {
|
|
@@ -1833,7 +1314,8 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1833
1314
|
return state;
|
|
1834
1315
|
},
|
|
1835
1316
|
async completeRun(report, artifactDir, reportPath) {
|
|
1836
|
-
|
|
1317
|
+
await ensure();
|
|
1318
|
+
if (await exists(cancellationPath(report.run_id))) {
|
|
1837
1319
|
return preserveCancellationRequestState(report.run_id);
|
|
1838
1320
|
}
|
|
1839
1321
|
const request = await readJson2(runRequestPath(report.run_id));
|
|
@@ -1845,7 +1327,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1845
1327
|
createdAt: completedAt
|
|
1846
1328
|
});
|
|
1847
1329
|
if (!model) {
|
|
1848
|
-
await rm(
|
|
1330
|
+
await rm(join3(paths.modelsDir, `local-${report.run_id}`), { recursive: true, force: true });
|
|
1849
1331
|
}
|
|
1850
1332
|
const previous = await getRun(report.run_id);
|
|
1851
1333
|
const state = {
|
|
@@ -1859,11 +1341,17 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1859
1341
|
completed_at: completedAt,
|
|
1860
1342
|
updated_at: completedAt
|
|
1861
1343
|
};
|
|
1344
|
+
if (!await exists(reportPath)) {
|
|
1345
|
+
throw new Error(`Run report not found: ${reportPath}`);
|
|
1346
|
+
}
|
|
1347
|
+
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
1348
|
+
if (await exists(cancellationPath(report.run_id))) {
|
|
1349
|
+
return preserveCancellationRequestState(report.run_id);
|
|
1350
|
+
}
|
|
1862
1351
|
await writeRunState(state);
|
|
1863
|
-
if (await
|
|
1352
|
+
if (await exists(cancellationPath(report.run_id))) {
|
|
1864
1353
|
return preserveCancellationRequestState(report.run_id);
|
|
1865
1354
|
}
|
|
1866
|
-
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
1867
1355
|
await appendRunEvent(state, {
|
|
1868
1356
|
stage: "completed",
|
|
1869
1357
|
status: "completed",
|
|
@@ -1877,7 +1365,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1877
1365
|
const previous = await getRun(runId);
|
|
1878
1366
|
if (options.report) await rm(runReportPath(previous.id), { force: true });
|
|
1879
1367
|
if (options.model) {
|
|
1880
|
-
await rm(
|
|
1368
|
+
await rm(join3(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true });
|
|
1881
1369
|
}
|
|
1882
1370
|
const state = {
|
|
1883
1371
|
...previous,
|
|
@@ -1892,7 +1380,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1892
1380
|
async failRun(runId, error) {
|
|
1893
1381
|
const previous = await getRun(runId);
|
|
1894
1382
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1895
|
-
if (await
|
|
1383
|
+
if (await exists(cancellationPath(previous.id))) {
|
|
1896
1384
|
return preserveCancellationRequestState(previous.id);
|
|
1897
1385
|
}
|
|
1898
1386
|
const state = {
|
|
@@ -1905,7 +1393,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1905
1393
|
updated_at: now
|
|
1906
1394
|
};
|
|
1907
1395
|
await writeRunState(state);
|
|
1908
|
-
if (await
|
|
1396
|
+
if (await exists(cancellationPath(previous.id))) {
|
|
1909
1397
|
return preserveCancellationRequestState(previous.id);
|
|
1910
1398
|
}
|
|
1911
1399
|
await appendRunEvent(state, { stage: "failed", status: "failed", message: error });
|
|
@@ -1916,8 +1404,8 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1916
1404
|
if (isTerminalRunState(state) || state.status === "cancelled" && state.current_stage === "cancel_requested") {
|
|
1917
1405
|
return;
|
|
1918
1406
|
}
|
|
1919
|
-
await
|
|
1920
|
-
|
|
1407
|
+
await writePrivateText(cancellationPath(state.id), `${(/* @__PURE__ */ new Date()).toISOString()}
|
|
1408
|
+
`);
|
|
1921
1409
|
const latest = await getRun(state.id);
|
|
1922
1410
|
if (isTerminalRunState(latest)) {
|
|
1923
1411
|
if (latest.status !== "cancelled") await rm(cancellationPath(state.id), { force: true });
|
|
@@ -1940,7 +1428,7 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1940
1428
|
},
|
|
1941
1429
|
async isCancellationRequested(runId) {
|
|
1942
1430
|
const state = await getRun(runId);
|
|
1943
|
-
return state.status === "cancelled" || await
|
|
1431
|
+
return state.status === "cancelled" || await exists(cancellationPath(state.id));
|
|
1944
1432
|
},
|
|
1945
1433
|
async listRuns() {
|
|
1946
1434
|
return listRunRecords();
|
|
@@ -1950,78 +1438,727 @@ function createLocalStore(root = defaultLocalHome()) {
|
|
|
1950
1438
|
const state = await getRun(id);
|
|
1951
1439
|
return readJsonl(runEventsPath(state.id));
|
|
1952
1440
|
},
|
|
1953
|
-
async getRunReport(id) {
|
|
1954
|
-
const state = await getRun(id);
|
|
1955
|
-
const candidates = [];
|
|
1956
|
-
if (state.report_path) {
|
|
1957
|
-
const reportPath =
|
|
1958
|
-
const allowedRoots = [resolvedRoot];
|
|
1959
|
-
if (state.artifact_dir) allowedRoots.push(
|
|
1960
|
-
if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) {
|
|
1961
|
-
candidates.push(reportPath);
|
|
1441
|
+
async getRunReport(id) {
|
|
1442
|
+
const state = await getRun(id);
|
|
1443
|
+
const candidates = [];
|
|
1444
|
+
if (state.report_path) {
|
|
1445
|
+
const reportPath = resolve4(state.report_path);
|
|
1446
|
+
const allowedRoots = [resolvedRoot];
|
|
1447
|
+
if (state.artifact_dir) allowedRoots.push(resolve4(state.artifact_dir));
|
|
1448
|
+
if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) {
|
|
1449
|
+
candidates.push(reportPath);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
candidates.push(runReportPath(state.id));
|
|
1453
|
+
for (const path of candidates) {
|
|
1454
|
+
if (await exists(path)) return readJson2(path);
|
|
1455
|
+
}
|
|
1456
|
+
throw new Error(`Run has no report yet: ${id}`);
|
|
1457
|
+
},
|
|
1458
|
+
async listModels() {
|
|
1459
|
+
return listModelRecords();
|
|
1460
|
+
},
|
|
1461
|
+
async getModel(id) {
|
|
1462
|
+
const record = findByIdOrPrefix(await listModelRecords(), id);
|
|
1463
|
+
if (!record) throw new Error(`Model not found: ${id}`);
|
|
1464
|
+
return record;
|
|
1465
|
+
}
|
|
1466
|
+
};
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
// src/local-runtime/contracts.ts
|
|
1470
|
+
import { z } from "zod";
|
|
1471
|
+
var datasetFormatSchema = z.literal("chat_jsonl");
|
|
1472
|
+
var baseModelRevisionSchema = z.string().regex(/^[0-9a-f]{40}$/i, "base_model_revision must be a 40-character Hugging Face commit SHA").transform((value) => value.toLowerCase());
|
|
1473
|
+
var behaviorSpecExampleSchema = z.object({
|
|
1474
|
+
input: z.string().min(1),
|
|
1475
|
+
output: z.string().min(1)
|
|
1476
|
+
}).strict();
|
|
1477
|
+
var specSnapshotSchema = z.object({
|
|
1478
|
+
name: z.string().min(1),
|
|
1479
|
+
description: z.string().default(""),
|
|
1480
|
+
system_prompt: z.string().default(""),
|
|
1481
|
+
guidelines: z.array(z.string()).default([]),
|
|
1482
|
+
examples: z.array(behaviorSpecExampleSchema).default([]),
|
|
1483
|
+
constraints: z.array(z.string()).default([]),
|
|
1484
|
+
base_model: z.string().transform((value) => canonicalizeTrainingModel(value))
|
|
1485
|
+
}).strict();
|
|
1486
|
+
var foundationHyperparametersSchema = z.object({
|
|
1487
|
+
vocab_size: z.number().int().min(64).max(65536).default(256),
|
|
1488
|
+
max_chars: z.number().int().min(1e3).max(2e9).default(2e4),
|
|
1489
|
+
depth: z.number().int().min(1).max(64).default(2),
|
|
1490
|
+
pretrain_steps: z.number().int().min(1).max(1e7).default(2),
|
|
1491
|
+
finetune_steps: z.number().int().min(1).max(1e7).default(2),
|
|
1492
|
+
rl_steps: z.number().int().min(0).max(1e7).default(0),
|
|
1493
|
+
batch_size: z.number().int().min(1).max(1024).default(2),
|
|
1494
|
+
sequence_length: z.number().int().min(16).max(8192).default(64),
|
|
1495
|
+
nproc_per_node: z.number().int().min(1).max(16).default(1)
|
|
1496
|
+
}).strict();
|
|
1497
|
+
var localFoundationSpecFileSchema = z.object({
|
|
1498
|
+
engine: z.literal("foundation"),
|
|
1499
|
+
id: z.string().uuid().optional(),
|
|
1500
|
+
name: z.string().min(1),
|
|
1501
|
+
description: z.string().default(""),
|
|
1502
|
+
system_prompt: z.string().default(""),
|
|
1503
|
+
guidelines: z.array(z.string()).default([]),
|
|
1504
|
+
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
1505
|
+
constraints: z.array(z.string()).default([]),
|
|
1506
|
+
foundation: foundationHyperparametersSchema
|
|
1507
|
+
}).strict();
|
|
1508
|
+
var datasetPrebuiltSchema = z.object({
|
|
1509
|
+
training: z.string().min(1),
|
|
1510
|
+
validation: z.string().min(1).optional(),
|
|
1511
|
+
test: z.string().min(1).optional(),
|
|
1512
|
+
format: datasetFormatSchema.default("chat_jsonl")
|
|
1513
|
+
}).strict().superRefine((dataset, context) => {
|
|
1514
|
+
if (!dataset.validation && !dataset.test) {
|
|
1515
|
+
context.addIssue({
|
|
1516
|
+
code: z.ZodIssueCode.custom,
|
|
1517
|
+
path: ["validation"],
|
|
1518
|
+
message: "Provide a distinct validation or test split for held-out evaluation"
|
|
1519
|
+
});
|
|
1520
|
+
}
|
|
1521
|
+
});
|
|
1522
|
+
var fineTuneHyperparametersSchema = z.object({
|
|
1523
|
+
n_epochs: z.number().int().min(1).max(20).default(1),
|
|
1524
|
+
learning_rate: z.number().positive().optional(),
|
|
1525
|
+
batch_size: z.number().int().min(1).optional(),
|
|
1526
|
+
lora_rank: z.number().int().min(1).optional(),
|
|
1527
|
+
lora_alpha: z.number().int().min(1).optional(),
|
|
1528
|
+
lora_dropout: z.number().min(0).max(1).optional(),
|
|
1529
|
+
max_seq_length: z.number().int().min(128).max(32768).optional(),
|
|
1530
|
+
gradient_accumulation_steps: z.number().int().min(1).optional(),
|
|
1531
|
+
max_eval_examples: z.number().int().min(1).optional(),
|
|
1532
|
+
/** Immutable Hugging Face commit used by every stage of the run. */
|
|
1533
|
+
base_model_revision: baseModelRevisionSchema.optional()
|
|
1534
|
+
}).strict();
|
|
1535
|
+
var fineTuneRunRequestSchema = z.object({
|
|
1536
|
+
run_id: z.string().uuid(),
|
|
1537
|
+
user_id: z.string().min(1),
|
|
1538
|
+
behavior_spec_id: z.string().uuid(),
|
|
1539
|
+
run_number: z.number().int().min(1),
|
|
1540
|
+
spec_snapshot: specSnapshotSchema,
|
|
1541
|
+
hyperparameters: fineTuneHyperparametersSchema.default({ n_epochs: 1 }),
|
|
1542
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
1543
|
+
}).strict().superRefine((request, context) => {
|
|
1544
|
+
if (request.spec_snapshot.examples.length === 0 && !request.dataset_prebuilt) {
|
|
1545
|
+
context.addIssue({
|
|
1546
|
+
code: z.ZodIssueCode.custom,
|
|
1547
|
+
path: ["spec_snapshot", "examples"],
|
|
1548
|
+
message: "Add examples or provide dataset_prebuilt training and validation files"
|
|
1549
|
+
});
|
|
1550
|
+
}
|
|
1551
|
+
});
|
|
1552
|
+
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
1553
|
+
engine: z.literal("adapter").optional(),
|
|
1554
|
+
id: z.string().uuid().optional(),
|
|
1555
|
+
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
1556
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
1557
|
+
}).strict();
|
|
1558
|
+
var localBehaviorSpecFileSchema = z.union([
|
|
1559
|
+
localFoundationSpecFileSchema,
|
|
1560
|
+
localAdapterSpecFileSchema
|
|
1561
|
+
]);
|
|
1562
|
+
var evalExampleResultSchema = z.object({
|
|
1563
|
+
prompt: z.string(),
|
|
1564
|
+
expected: z.string(),
|
|
1565
|
+
actual: z.string(),
|
|
1566
|
+
passed: z.boolean(),
|
|
1567
|
+
score: z.number().min(0).max(1),
|
|
1568
|
+
reasoning: z.string().nullable(),
|
|
1569
|
+
latency_ms: z.number().int().nonnegative(),
|
|
1570
|
+
scored_by: z.enum(["exact_match", "json_fields", "heuristic"]).optional()
|
|
1571
|
+
}).strict();
|
|
1572
|
+
var jsonFieldMetricsSchema = z.object({
|
|
1573
|
+
fields: z.array(z.string()),
|
|
1574
|
+
valid_json_count: z.number().int().nonnegative(),
|
|
1575
|
+
valid_json_rate: z.number().min(0).max(1),
|
|
1576
|
+
schema_match_count: z.number().int().nonnegative(),
|
|
1577
|
+
schema_match_rate: z.number().min(0).max(1),
|
|
1578
|
+
all_fields_match_count: z.number().int().nonnegative(),
|
|
1579
|
+
all_fields_match_rate: z.number().min(0).max(1),
|
|
1580
|
+
field_accuracy: z.record(z.string(), z.object({
|
|
1581
|
+
correct: z.number().int().nonnegative(),
|
|
1582
|
+
total: z.number().int().nonnegative(),
|
|
1583
|
+
accuracy: z.number().min(0).max(1)
|
|
1584
|
+
}).strict())
|
|
1585
|
+
}).strict();
|
|
1586
|
+
var evalSplitSchema = z.enum([
|
|
1587
|
+
"spec_examples",
|
|
1588
|
+
"spec_holdout",
|
|
1589
|
+
"prebuilt_test",
|
|
1590
|
+
"prebuilt_validation",
|
|
1591
|
+
"general_regression"
|
|
1592
|
+
]);
|
|
1593
|
+
var evalReportSchema = z.object({
|
|
1594
|
+
kind: z.enum(["baseline", "candidate"]),
|
|
1595
|
+
model_id: z.string(),
|
|
1596
|
+
total: z.number().int().nonnegative(),
|
|
1597
|
+
eval_examples_total: z.number().int().nonnegative(),
|
|
1598
|
+
eval_examples_used: z.number().int().nonnegative(),
|
|
1599
|
+
eval_truncated: z.boolean(),
|
|
1600
|
+
eval_split: evalSplitSchema.optional(),
|
|
1601
|
+
eval_sample_seed: z.number().int().nullable().optional(),
|
|
1602
|
+
avg_score: z.number().min(0).max(1),
|
|
1603
|
+
pass_rate: z.number().min(0).max(1),
|
|
1604
|
+
exact_match_rate: z.number().min(0).max(1),
|
|
1605
|
+
avg_token_f1: z.number().min(0).max(1).optional(),
|
|
1606
|
+
avg_latency_ms: z.number().int().nonnegative(),
|
|
1607
|
+
cached: z.boolean().optional(),
|
|
1608
|
+
cache_key: z.string().optional(),
|
|
1609
|
+
results: z.array(evalExampleResultSchema),
|
|
1610
|
+
artifact_uri: z.string(),
|
|
1611
|
+
scoring_method: z.enum(["heuristic", "exact_match", "json_fields"]),
|
|
1612
|
+
inference_provider: z.enum(["none", "transformers"]).optional(),
|
|
1613
|
+
scoring_mode: z.enum(["exact_match", "json_fields"]).optional(),
|
|
1614
|
+
json_field_metrics: jsonFieldMetricsSchema.optional(),
|
|
1615
|
+
generation_config: z.record(z.string(), z.unknown()).optional(),
|
|
1616
|
+
log_uri: z.string().optional()
|
|
1617
|
+
}).strict();
|
|
1618
|
+
var comparisonReportSchema = z.object({
|
|
1619
|
+
avg_score_delta: z.number(),
|
|
1620
|
+
pass_rate_delta: z.number(),
|
|
1621
|
+
exact_match_rate_delta: z.number(),
|
|
1622
|
+
token_f1_delta: z.number().optional(),
|
|
1623
|
+
regressions: z.number().int().nonnegative(),
|
|
1624
|
+
improvements: z.number().int().nonnegative(),
|
|
1625
|
+
regressed_examples: z.array(z.object({
|
|
1626
|
+
prompt: z.string(),
|
|
1627
|
+
old_score: z.number(),
|
|
1628
|
+
new_score: z.number()
|
|
1629
|
+
}).strict())
|
|
1630
|
+
}).strict();
|
|
1631
|
+
var trainingReportSchema = z.object({
|
|
1632
|
+
provider: z.literal("local-uv"),
|
|
1633
|
+
training_job_name: z.string(),
|
|
1634
|
+
model_artifact_uri: z.string().optional(),
|
|
1635
|
+
base_model_artifact_uri: z.string().optional(),
|
|
1636
|
+
metrics: z.record(z.string(), z.unknown()).nullable(),
|
|
1637
|
+
exit_code: z.number().int().nullable(),
|
|
1638
|
+
log_uri: z.string(),
|
|
1639
|
+
command: z.array(z.string()).optional()
|
|
1640
|
+
}).strict();
|
|
1641
|
+
var runReportSchema = z.object({
|
|
1642
|
+
run_id: z.string().uuid(),
|
|
1643
|
+
behavior_spec_id: z.string().uuid(),
|
|
1644
|
+
user_id: z.string(),
|
|
1645
|
+
run_number: z.number().int().min(1),
|
|
1646
|
+
base_model: z.string(),
|
|
1647
|
+
fine_tuned_model_id: z.string(),
|
|
1648
|
+
status: z.enum(["completed", "failed"]),
|
|
1649
|
+
baseline: evalReportSchema,
|
|
1650
|
+
candidate: evalReportSchema,
|
|
1651
|
+
comparison: comparisonReportSchema,
|
|
1652
|
+
general_regression: z.object({
|
|
1653
|
+
dataset_uri: z.string(),
|
|
1654
|
+
dataset_sha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
1655
|
+
baseline: evalReportSchema,
|
|
1656
|
+
candidate: evalReportSchema,
|
|
1657
|
+
comparison: comparisonReportSchema,
|
|
1658
|
+
policy: z.object({
|
|
1659
|
+
max_score_drop: z.number().min(0).max(1),
|
|
1660
|
+
max_pass_rate_drop: z.number().min(0).max(1)
|
|
1661
|
+
}).strict(),
|
|
1662
|
+
passed: z.boolean(),
|
|
1663
|
+
failures: z.array(z.string())
|
|
1664
|
+
}).strict().optional(),
|
|
1665
|
+
training: trainingReportSchema,
|
|
1666
|
+
artifact_uris: z.object({
|
|
1667
|
+
dataset: z.string(),
|
|
1668
|
+
baseline_eval: z.string(),
|
|
1669
|
+
candidate_eval: z.string(),
|
|
1670
|
+
general_baseline_eval: z.string().optional(),
|
|
1671
|
+
general_candidate_eval: z.string().optional(),
|
|
1672
|
+
report: z.string()
|
|
1673
|
+
}).strict(),
|
|
1674
|
+
run_metadata: z.object({
|
|
1675
|
+
base_model: z.string(),
|
|
1676
|
+
fine_tuned_model_id: z.string(),
|
|
1677
|
+
dataset_prebuilt: z.boolean(),
|
|
1678
|
+
dataset_format: datasetFormatSchema.nullable().optional(),
|
|
1679
|
+
dataset_uri: z.string(),
|
|
1680
|
+
spec_example_count: z.number().int().nonnegative(),
|
|
1681
|
+
training_example_count: z.number().int().nonnegative().nullable(),
|
|
1682
|
+
eval_examples_total: z.number().int().nonnegative(),
|
|
1683
|
+
eval_examples_used: z.number().int().nonnegative(),
|
|
1684
|
+
eval_split: evalSplitSchema.optional(),
|
|
1685
|
+
eval_sample_seed: z.number().int().nullable().optional(),
|
|
1686
|
+
started_at: z.string(),
|
|
1687
|
+
completed_at: z.string(),
|
|
1688
|
+
elapsed_ms: z.number().int().nonnegative(),
|
|
1689
|
+
elapsed_seconds: z.number().nonnegative()
|
|
1690
|
+
}).strict(),
|
|
1691
|
+
created_at: z.string()
|
|
1692
|
+
}).strict();
|
|
1693
|
+
var inferenceConfigSchema = z.object({
|
|
1694
|
+
maxNewTokens: z.number().int().min(1).default(256),
|
|
1695
|
+
temperature: z.number().min(0).default(0),
|
|
1696
|
+
topP: z.number().min(0).max(1).default(1),
|
|
1697
|
+
device: z.enum(["cuda", "cpu"]).default("cuda")
|
|
1698
|
+
}).strict();
|
|
1699
|
+
var scoringConfigSchema = z.object({
|
|
1700
|
+
mode: z.enum(["exact_match", "json_fields"]).default("exact_match"),
|
|
1701
|
+
fields: z.array(z.string().min(1)).optional()
|
|
1702
|
+
}).strict();
|
|
1703
|
+
var evaluationConfigSchema = z.object({
|
|
1704
|
+
inference: inferenceConfigSchema.default({
|
|
1705
|
+
maxNewTokens: 256,
|
|
1706
|
+
temperature: 0,
|
|
1707
|
+
topP: 1,
|
|
1708
|
+
device: "cuda"
|
|
1709
|
+
}),
|
|
1710
|
+
scoring: scoringConfigSchema.default({ mode: "exact_match" }),
|
|
1711
|
+
timeoutMs: z.number().int().min(100).default(18e5),
|
|
1712
|
+
maxExamples: z.number().int().min(1).optional(),
|
|
1713
|
+
sampleSeed: z.number().int().optional(),
|
|
1714
|
+
baselineCache: z.boolean().default(true),
|
|
1715
|
+
generalRegression: z.object({
|
|
1716
|
+
dataset: z.string().min(1),
|
|
1717
|
+
systemPrompt: z.string().optional(),
|
|
1718
|
+
maxScoreDrop: z.number().min(0).max(1).default(0.03),
|
|
1719
|
+
maxPassRateDrop: z.number().min(0).max(1).default(0.05)
|
|
1720
|
+
}).strict().optional()
|
|
1721
|
+
}).strict();
|
|
1722
|
+
var localRunnerConfigSchema = z.object({
|
|
1723
|
+
storeRoot: z.string().optional(),
|
|
1724
|
+
artifactRoot: z.string().default(DEFAULT_ARTIFACT_ROOT),
|
|
1725
|
+
dryRun: z.boolean().default(false),
|
|
1726
|
+
paths: z.object({
|
|
1727
|
+
baseModel: z.string().optional(),
|
|
1728
|
+
modelCache: z.string().optional()
|
|
1729
|
+
}).strict().default({}),
|
|
1730
|
+
evaluation: evaluationConfigSchema.default({
|
|
1731
|
+
inference: {
|
|
1732
|
+
maxNewTokens: 256,
|
|
1733
|
+
temperature: 0,
|
|
1734
|
+
topP: 1,
|
|
1735
|
+
device: "cuda"
|
|
1736
|
+
},
|
|
1737
|
+
scoring: { mode: "exact_match" },
|
|
1738
|
+
timeoutMs: 18e5,
|
|
1739
|
+
baselineCache: true
|
|
1740
|
+
})
|
|
1741
|
+
}).strict();
|
|
1742
|
+
function isFoundationSpecFile(spec) {
|
|
1743
|
+
return spec.engine === "foundation";
|
|
1744
|
+
}
|
|
1745
|
+
|
|
1746
|
+
// src/local-runtime/local-project.ts
|
|
1747
|
+
import { createHash as createHash2, randomUUID as randomUUID3 } from "crypto";
|
|
1748
|
+
import { readFile as readFile3, stat as stat3, writeFile as writeFile3 } from "fs/promises";
|
|
1749
|
+
import { dirname as dirname3, isAbsolute as isAbsolute3, resolve as resolve5 } from "path";
|
|
1750
|
+
import { homedir as homedir2 } from "os";
|
|
1751
|
+
import { mkdir as mkdir3 } from "fs/promises";
|
|
1752
|
+
var DEFAULT_LOCAL_SPEC_PATH = "tunedtensor.json";
|
|
1753
|
+
function resolveLocalReference(value, baseDirectory) {
|
|
1754
|
+
if (typeof value !== "string" || !value || isAbsolute3(value) || /^[a-z][a-z0-9+.-]*:/i.test(value)) {
|
|
1755
|
+
return value;
|
|
1756
|
+
}
|
|
1757
|
+
if (value === "~") return homedir2();
|
|
1758
|
+
if (value.startsWith("~/")) return resolve5(homedir2(), value.slice(2));
|
|
1759
|
+
return resolve5(baseDirectory, value);
|
|
1760
|
+
}
|
|
1761
|
+
function resolveLocalRunInputPaths(raw, inputPath) {
|
|
1762
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw;
|
|
1763
|
+
const baseDirectory = dirname3(resolve5(inputPath));
|
|
1764
|
+
const value = structuredClone(raw);
|
|
1765
|
+
const dataset = value.dataset_prebuilt;
|
|
1766
|
+
if (dataset && typeof dataset === "object" && !Array.isArray(dataset)) {
|
|
1767
|
+
const fields = dataset;
|
|
1768
|
+
for (const key of ["training", "validation", "test"]) {
|
|
1769
|
+
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
return value;
|
|
1773
|
+
}
|
|
1774
|
+
async function exists2(path) {
|
|
1775
|
+
try {
|
|
1776
|
+
await stat3(path);
|
|
1777
|
+
return true;
|
|
1778
|
+
} catch {
|
|
1779
|
+
return false;
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1782
|
+
async function initLocalSpecFile(args) {
|
|
1783
|
+
if (!args.force && await exists2(args.outputPath)) {
|
|
1784
|
+
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
1785
|
+
}
|
|
1786
|
+
if (args.engine === "foundation") {
|
|
1787
|
+
const spec2 = localFoundationSpecFileSchema.parse({
|
|
1788
|
+
engine: "foundation",
|
|
1789
|
+
id: randomUUID3(),
|
|
1790
|
+
name: args.name,
|
|
1791
|
+
description: "From-scratch tokenizer, pretrain, chat SFT, and eval.",
|
|
1792
|
+
system_prompt: "You are a helpful assistant.",
|
|
1793
|
+
guidelines: [
|
|
1794
|
+
"Answer the user request directly."
|
|
1795
|
+
],
|
|
1796
|
+
constraints: [],
|
|
1797
|
+
examples: [
|
|
1798
|
+
{
|
|
1799
|
+
input: "Replace this with representative chat input.",
|
|
1800
|
+
output: "Replace this with the expected assistant reply."
|
|
1801
|
+
},
|
|
1802
|
+
{
|
|
1803
|
+
input: "Replace this with a different chat input.",
|
|
1804
|
+
output: "Replace this with the expected assistant reply."
|
|
1962
1805
|
}
|
|
1806
|
+
],
|
|
1807
|
+
foundation: {
|
|
1808
|
+
depth: 2,
|
|
1809
|
+
pretrain_steps: 2,
|
|
1810
|
+
finetune_steps: 2,
|
|
1811
|
+
rl_steps: 0,
|
|
1812
|
+
vocab_size: 256,
|
|
1813
|
+
max_chars: 2e4,
|
|
1814
|
+
sequence_length: 64,
|
|
1815
|
+
batch_size: 2,
|
|
1816
|
+
nproc_per_node: 1
|
|
1963
1817
|
}
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1818
|
+
});
|
|
1819
|
+
await mkdir3(dirname3(args.outputPath), { recursive: true });
|
|
1820
|
+
await writeFile3(args.outputPath, `${JSON.stringify(spec2, null, 2)}
|
|
1821
|
+
`, "utf8");
|
|
1822
|
+
return spec2;
|
|
1823
|
+
}
|
|
1824
|
+
if (!args.baseModel) {
|
|
1825
|
+
throw new Error("Adapter specs require --model.");
|
|
1826
|
+
}
|
|
1827
|
+
const spec = localAdapterSpecFileSchema.parse({
|
|
1828
|
+
id: randomUUID3(),
|
|
1829
|
+
name: args.name,
|
|
1830
|
+
description: "",
|
|
1831
|
+
system_prompt: "Describe the behavior this local model should learn.",
|
|
1832
|
+
guidelines: [
|
|
1833
|
+
"Return concise, task-specific answers."
|
|
1834
|
+
],
|
|
1835
|
+
constraints: [],
|
|
1836
|
+
base_model: args.baseModel,
|
|
1837
|
+
examples: [
|
|
1838
|
+
{
|
|
1839
|
+
input: "Replace this with representative training input.",
|
|
1840
|
+
output: "Replace this with the expected training output."
|
|
1841
|
+
},
|
|
1842
|
+
{
|
|
1843
|
+
input: "Replace this with a different validation input.",
|
|
1844
|
+
output: "Replace this with the expected validation output."
|
|
1967
1845
|
}
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1846
|
+
],
|
|
1847
|
+
hyperparameters: {
|
|
1848
|
+
n_epochs: 1
|
|
1849
|
+
}
|
|
1850
|
+
});
|
|
1851
|
+
await mkdir3(dirname3(args.outputPath), { recursive: true });
|
|
1852
|
+
await writeFile3(args.outputPath, `${JSON.stringify(spec, null, 2)}
|
|
1853
|
+
`, "utf8");
|
|
1854
|
+
return spec;
|
|
1855
|
+
}
|
|
1856
|
+
async function initLocalRunnerConfigFile(args) {
|
|
1857
|
+
if (!args.force && await exists2(args.outputPath)) {
|
|
1858
|
+
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
1859
|
+
}
|
|
1860
|
+
const config = {
|
|
1861
|
+
artifactRoot: DEFAULT_ARTIFACT_ROOT,
|
|
1862
|
+
storeRoot: DEFAULT_PROJECT_STORE_ROOT,
|
|
1863
|
+
evaluation: {
|
|
1864
|
+
inference: {
|
|
1865
|
+
device: "cuda"
|
|
1866
|
+
},
|
|
1867
|
+
scoring: {
|
|
1868
|
+
mode: "exact_match"
|
|
1869
|
+
},
|
|
1870
|
+
timeoutMs: 18e5
|
|
1871
|
+
}
|
|
1872
|
+
};
|
|
1873
|
+
await mkdir3(dirname3(args.outputPath), { recursive: true });
|
|
1874
|
+
await writeFile3(args.outputPath, `${JSON.stringify(config, null, 2)}
|
|
1875
|
+
`, "utf8");
|
|
1876
|
+
return config;
|
|
1877
|
+
}
|
|
1878
|
+
function placeholderIssues(systemPrompt, examples) {
|
|
1879
|
+
const issues = [];
|
|
1880
|
+
if (/describe the behavior this local model should learn/i.test(systemPrompt)) {
|
|
1881
|
+
issues.push("system_prompt still contains the generated placeholder");
|
|
1882
|
+
}
|
|
1883
|
+
examples.forEach((example, index) => {
|
|
1884
|
+
if (/replace this with/i.test(example.input) || /replace this with/i.test(example.output)) {
|
|
1885
|
+
issues.push(`examples[${index}] still contains generated placeholder text`);
|
|
1886
|
+
}
|
|
1887
|
+
});
|
|
1888
|
+
return issues;
|
|
1889
|
+
}
|
|
1890
|
+
function generatedPlaceholderIssues(request) {
|
|
1891
|
+
return placeholderIssues(request.spec_snapshot.system_prompt, request.spec_snapshot.examples);
|
|
1892
|
+
}
|
|
1893
|
+
function assertLocalRunInputReady(request) {
|
|
1894
|
+
const issues = generatedPlaceholderIssues(request);
|
|
1895
|
+
if (issues.length > 0) {
|
|
1896
|
+
throw new Error(`Edit the generated behavior spec before training: ${issues.join("; ")}.`);
|
|
1897
|
+
}
|
|
1898
|
+
}
|
|
1899
|
+
function foundationPlaceholderIssues(spec) {
|
|
1900
|
+
return placeholderIssues(spec.system_prompt, spec.examples);
|
|
1901
|
+
}
|
|
1902
|
+
function assertFoundationSpecReady(spec) {
|
|
1903
|
+
const issues = foundationPlaceholderIssues(spec);
|
|
1904
|
+
if (issues.length > 0) {
|
|
1905
|
+
throw new Error(`Edit the generated foundation spec before training: ${issues.join("; ")}.`);
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
function runRequestFromLocalSpec(spec, options = {}) {
|
|
1909
|
+
if (isFoundationSpecFile(spec)) {
|
|
1910
|
+
throw new Error(
|
|
1911
|
+
"Foundation specs cannot be converted into a LoRA run request. Use `tt pipeline run --spec`."
|
|
1912
|
+
);
|
|
1913
|
+
}
|
|
1914
|
+
const {
|
|
1915
|
+
id,
|
|
1916
|
+
engine: _engine,
|
|
1917
|
+
hyperparameters,
|
|
1918
|
+
dataset_prebuilt: datasetPrebuilt,
|
|
1919
|
+
...specSnapshot
|
|
1920
|
+
} = spec;
|
|
1921
|
+
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
1922
|
+
return fineTuneRunRequestSchema.parse({
|
|
1923
|
+
run_id: options.runId ?? randomUUID3(),
|
|
1924
|
+
user_id: "local-user",
|
|
1925
|
+
behavior_spec_id: behaviorSpecId,
|
|
1926
|
+
run_number: 1,
|
|
1927
|
+
spec_snapshot: specSnapshot,
|
|
1928
|
+
hyperparameters,
|
|
1929
|
+
dataset_prebuilt: datasetPrebuilt
|
|
1930
|
+
});
|
|
1931
|
+
}
|
|
1932
|
+
function deterministicBehaviorSpecId(spec) {
|
|
1933
|
+
const namespace = Buffer.from("9f783f01340457bea061942004259253", "hex");
|
|
1934
|
+
const digest = createHash2("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
|
|
1935
|
+
digest[6] = digest[6] & 15 | 80;
|
|
1936
|
+
digest[8] = digest[8] & 63 | 128;
|
|
1937
|
+
const hex = digest.toString("hex");
|
|
1938
|
+
return [
|
|
1939
|
+
hex.slice(0, 8),
|
|
1940
|
+
hex.slice(8, 12),
|
|
1941
|
+
hex.slice(12, 16),
|
|
1942
|
+
hex.slice(16, 20),
|
|
1943
|
+
hex.slice(20)
|
|
1944
|
+
].join("-");
|
|
1945
|
+
}
|
|
1946
|
+
async function loadLocalRunInput(path, options = {}) {
|
|
1947
|
+
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile3(path, "utf8")), path);
|
|
1948
|
+
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
1949
|
+
if (request.success) {
|
|
1950
|
+
return { kind: "request", path, request: request.data };
|
|
1951
|
+
}
|
|
1952
|
+
const spec = localBehaviorSpecFileSchema.parse(raw);
|
|
1953
|
+
if (isFoundationSpecFile(spec)) {
|
|
1954
|
+
return { kind: "foundation-spec", path, spec };
|
|
1955
|
+
}
|
|
1956
|
+
return {
|
|
1957
|
+
kind: "spec",
|
|
1958
|
+
path,
|
|
1959
|
+
spec,
|
|
1960
|
+
request: runRequestFromLocalSpec(spec, options)
|
|
1961
|
+
};
|
|
1962
|
+
}
|
|
1963
|
+
|
|
1964
|
+
// src/local-runtime/dataset.ts
|
|
1965
|
+
import { readFile as readFile4 } from "fs/promises";
|
|
1966
|
+
async function loadNormalizedChatJsonl(path) {
|
|
1967
|
+
const text = await readFile4(path, "utf8");
|
|
1968
|
+
const rows = [];
|
|
1969
|
+
const jsonLines = [];
|
|
1970
|
+
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
1971
|
+
if (!line.trim()) continue;
|
|
1972
|
+
let value;
|
|
1973
|
+
try {
|
|
1974
|
+
value = JSON.parse(line);
|
|
1975
|
+
} catch (error) {
|
|
1976
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: malformed JSON`, { cause: error });
|
|
1977
|
+
}
|
|
1978
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1979
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: expected an object`);
|
|
1980
|
+
}
|
|
1981
|
+
const messages = value.messages;
|
|
1982
|
+
if (!Array.isArray(messages) || ![2, 3].includes(messages.length) || messages.some(
|
|
1983
|
+
(message) => !message || typeof message !== "object" || Array.isArray(message) || !["system", "user", "assistant"].includes(String(message.role)) || typeof message.content !== "string"
|
|
1984
|
+
)) {
|
|
1985
|
+
throw new Error(
|
|
1986
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one user message, and one assistant answer`
|
|
1987
|
+
);
|
|
1988
|
+
}
|
|
1989
|
+
const row = { messages };
|
|
1990
|
+
const offset = row.messages.length === 3 ? 1 : 0;
|
|
1991
|
+
if (offset === 1 && row.messages[0]?.role !== "system" || row.messages[offset]?.role !== "user" || !row.messages[offset]?.content.trim() || row.messages[offset + 1]?.role !== "assistant" || !row.messages[offset + 1]?.content.trim()) {
|
|
1992
|
+
throw new Error(
|
|
1993
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one non-empty user message, and one non-empty assistant answer`
|
|
1994
|
+
);
|
|
1995
|
+
}
|
|
1996
|
+
rows.push(row);
|
|
1997
|
+
jsonLines.push(JSON.stringify(row));
|
|
1998
|
+
}
|
|
1999
|
+
if (rows.length === 0) throw new Error(`Chat JSONL contains no examples: ${path}`);
|
|
2000
|
+
return {
|
|
2001
|
+
rows,
|
|
2002
|
+
jsonl: jsonLines.join("\n")
|
|
2003
|
+
};
|
|
2004
|
+
}
|
|
2005
|
+
async function normalizeChatJsonlForRelocation(path) {
|
|
2006
|
+
return (await loadNormalizedChatJsonl(path)).jsonl;
|
|
2007
|
+
}
|
|
2008
|
+
function buildSystemMessage(spec) {
|
|
2009
|
+
const parts = [];
|
|
2010
|
+
if (spec.system_prompt.trim()) parts.push(spec.system_prompt.trim());
|
|
2011
|
+
if (spec.guidelines.length > 0) {
|
|
2012
|
+
parts.push(`Guidelines:
|
|
2013
|
+
${spec.guidelines.map((guideline) => `- ${guideline}`).join("\n")}`);
|
|
2014
|
+
}
|
|
2015
|
+
if (spec.constraints.length > 0) {
|
|
2016
|
+
parts.push(`Constraints:
|
|
2017
|
+
${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
|
|
2018
|
+
}
|
|
2019
|
+
return parts.join("\n\n") || "Follow the demonstrated behavior.";
|
|
2020
|
+
}
|
|
2021
|
+
function exampleToChatRow(spec, example) {
|
|
2022
|
+
const system = buildSystemMessage(spec);
|
|
2023
|
+
return {
|
|
2024
|
+
messages: [
|
|
2025
|
+
{ role: "system", content: system },
|
|
2026
|
+
{ role: "user", content: example.input },
|
|
2027
|
+
{ role: "assistant", content: example.output }
|
|
2028
|
+
]
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
function compileSpecToJsonl(spec) {
|
|
2032
|
+
return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
|
|
2033
|
+
}
|
|
2034
|
+
function examplesFromSpec(spec) {
|
|
2035
|
+
return spec.examples;
|
|
2036
|
+
}
|
|
2037
|
+
async function examplesFromChatJsonl(path) {
|
|
2038
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
2039
|
+
const examples = [];
|
|
2040
|
+
for (const [index, row] of normalized.rows.entries()) {
|
|
2041
|
+
const user = row.messages.at(-2);
|
|
2042
|
+
const assistant = row.messages.at(-1);
|
|
2043
|
+
examples.push({
|
|
2044
|
+
input: user.content,
|
|
2045
|
+
output: assistant.content
|
|
2046
|
+
});
|
|
2047
|
+
}
|
|
2048
|
+
return examples;
|
|
2049
|
+
}
|
|
2050
|
+
async function evaluationSuiteFromChatJsonl(path, configuredSystem) {
|
|
2051
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
2052
|
+
const inputIdentities = /* @__PURE__ */ new Set();
|
|
2053
|
+
for (const row of normalized.rows) {
|
|
2054
|
+
const input = row.messages.at(-2).content;
|
|
2055
|
+
const identity = input.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2056
|
+
if (inputIdentities.has(identity)) {
|
|
2057
|
+
throw new Error(
|
|
2058
|
+
"General regression data contains duplicate inputs; evaluation prompts must be unique."
|
|
2059
|
+
);
|
|
1977
2060
|
}
|
|
2061
|
+
inputIdentities.add(identity);
|
|
2062
|
+
}
|
|
2063
|
+
const systems = new Set(
|
|
2064
|
+
normalized.rows.map((row) => row.messages.find((message) => message.role === "system")?.content).filter((value) => value !== void 0)
|
|
2065
|
+
);
|
|
2066
|
+
if (systems.size > 1) {
|
|
2067
|
+
throw new Error("General regression rows must use one consistent system message.");
|
|
2068
|
+
}
|
|
2069
|
+
const embeddedSystem = [...systems][0];
|
|
2070
|
+
if (configuredSystem !== void 0 && embeddedSystem !== void 0 && configuredSystem !== embeddedSystem) {
|
|
2071
|
+
throw new Error(
|
|
2072
|
+
"evaluation.generalRegression.systemPrompt must match the system message embedded in the dataset."
|
|
2073
|
+
);
|
|
2074
|
+
}
|
|
2075
|
+
return {
|
|
2076
|
+
examples: normalized.rows.map((row) => ({
|
|
2077
|
+
input: row.messages.at(-2).content,
|
|
2078
|
+
output: row.messages.at(-1).content
|
|
2079
|
+
})),
|
|
2080
|
+
system: configuredSystem ?? embeddedSystem ?? "You are a helpful assistant."
|
|
1978
2081
|
};
|
|
1979
2082
|
}
|
|
1980
2083
|
|
|
2084
|
+
// src/local-runtime/run-reporter.ts
|
|
2085
|
+
function reportInBackground(action) {
|
|
2086
|
+
try {
|
|
2087
|
+
void Promise.resolve(action()).catch(() => void 0);
|
|
2088
|
+
} catch {
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
2091
|
+
function sanitizeLogLine(line) {
|
|
2092
|
+
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
2093
|
+
}
|
|
2094
|
+
function forwardStreamLines(stream, onLine) {
|
|
2095
|
+
let buffered = "";
|
|
2096
|
+
stream.setEncoding("utf8");
|
|
2097
|
+
stream.on("data", (chunk) => {
|
|
2098
|
+
buffered += chunk;
|
|
2099
|
+
const lines = buffered.split(/\r\n|\n|\r/);
|
|
2100
|
+
buffered = lines.pop() ?? "";
|
|
2101
|
+
for (const line of lines) {
|
|
2102
|
+
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
2103
|
+
}
|
|
2104
|
+
});
|
|
2105
|
+
stream.on("end", () => {
|
|
2106
|
+
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
2107
|
+
});
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
// src/local-runtime/package-root.ts
|
|
2111
|
+
import { existsSync as existsSync2 } from "fs";
|
|
2112
|
+
import { dirname as dirname4, resolve as resolve6 } from "path";
|
|
2113
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
2114
|
+
function localRuntimePackageRoot(moduleUrl) {
|
|
2115
|
+
const moduleDirectory = dirname4(fileURLToPath2(moduleUrl));
|
|
2116
|
+
const candidates = [
|
|
2117
|
+
dirname4(moduleDirectory),
|
|
2118
|
+
dirname4(dirname4(moduleDirectory))
|
|
2119
|
+
];
|
|
2120
|
+
const root = candidates.find(
|
|
2121
|
+
(candidate) => existsSync2(resolve6(candidate, "package.json")) && existsSync2(resolve6(candidate, "training/adapter"))
|
|
2122
|
+
);
|
|
2123
|
+
if (!root) {
|
|
2124
|
+
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
2125
|
+
}
|
|
2126
|
+
return root;
|
|
2127
|
+
}
|
|
2128
|
+
|
|
1981
2129
|
// src/local-runtime/prefetch.ts
|
|
1982
2130
|
import { createHash as createHash4 } from "crypto";
|
|
1983
2131
|
import { createReadStream as createReadStream2 } from "fs";
|
|
1984
|
-
import { lstat as
|
|
1985
|
-
import { isAbsolute as
|
|
2132
|
+
import { lstat as lstat4, mkdir as mkdir5, readFile as readFile5, readdir as readdir4, realpath as realpath2, stat as stat4 } from "fs/promises";
|
|
2133
|
+
import { isAbsolute as isAbsolute4, join as join6, relative as relative2, resolve as resolve9, sep as sep3 } from "path";
|
|
1986
2134
|
|
|
1987
2135
|
// src/local-runtime/process-runner.ts
|
|
1988
2136
|
import { createHash as createHash3 } from "crypto";
|
|
1989
2137
|
import { spawn } from "child_process";
|
|
1990
2138
|
import { createWriteStream, readFileSync } from "fs";
|
|
1991
2139
|
import { mkdir as mkdir4 } from "fs/promises";
|
|
1992
|
-
import {
|
|
1993
|
-
import { dirname as dirname5, join as join3, resolve as resolve6 } from "path";
|
|
2140
|
+
import { dirname as dirname5, join as join4, resolve as resolve7 } from "path";
|
|
1994
2141
|
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
1995
|
-
var bundledProject =
|
|
2142
|
+
var bundledProject = join4(packageRoot, "training/adapter");
|
|
1996
2143
|
var bundledRuntimeHash = (() => {
|
|
1997
2144
|
const hash = createHash3("sha256");
|
|
1998
2145
|
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1999
|
-
hash.update(readFileSync(
|
|
2146
|
+
hash.update(readFileSync(join4(bundledProject, name)));
|
|
2000
2147
|
}
|
|
2001
2148
|
return hash.digest("hex").slice(0, 20);
|
|
2002
2149
|
})();
|
|
2003
|
-
var BUNDLED_PYTHON_ENVIRONMENT =
|
|
2004
|
-
|
|
2005
|
-
"tuned-tensor-local",
|
|
2006
|
-
"uv",
|
|
2007
|
-
bundledRuntimeHash
|
|
2008
|
-
);
|
|
2009
|
-
var foundationProject = join3(packageRoot, "training/foundation");
|
|
2150
|
+
var BUNDLED_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv", bundledRuntimeHash);
|
|
2151
|
+
var foundationProject = join4(packageRoot, "training/foundation");
|
|
2010
2152
|
var foundationRuntimeHash = (() => {
|
|
2011
2153
|
const hash = createHash3("sha256");
|
|
2012
2154
|
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
2013
|
-
hash.update(readFileSync(
|
|
2155
|
+
hash.update(readFileSync(join4(foundationProject, name)));
|
|
2014
2156
|
}
|
|
2015
2157
|
return hash.digest("hex").slice(0, 20);
|
|
2016
2158
|
})();
|
|
2017
|
-
var FOUNDATION_PYTHON_ENVIRONMENT =
|
|
2018
|
-
resolve6(process.env.XDG_CACHE_HOME ?? join3(homedir3(), ".cache")),
|
|
2019
|
-
"tuned-tensor-local",
|
|
2020
|
-
"uv-foundation",
|
|
2021
|
-
foundationRuntimeHash
|
|
2022
|
-
);
|
|
2159
|
+
var FOUNDATION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-foundation", foundationRuntimeHash);
|
|
2023
2160
|
function buildBundledPythonCommand(entrypoint, args = []) {
|
|
2024
|
-
const target = entrypoint === "-c" ? entrypoint :
|
|
2161
|
+
const target = entrypoint === "-c" ? entrypoint : join4(bundledProject, "src", entrypoint);
|
|
2025
2162
|
const commandArgs = [
|
|
2026
2163
|
"run",
|
|
2027
2164
|
"--frozen",
|
|
@@ -2052,7 +2189,7 @@ function buildFoundationPythonCommand(entrypoint, args = []) {
|
|
|
2052
2189
|
"--project",
|
|
2053
2190
|
foundationProject,
|
|
2054
2191
|
"python",
|
|
2055
|
-
|
|
2192
|
+
join4(foundationProject, "src", entrypoint),
|
|
2056
2193
|
...args
|
|
2057
2194
|
];
|
|
2058
2195
|
return {
|
|
@@ -2099,7 +2236,7 @@ async function runLoggedProcess(args) {
|
|
|
2099
2236
|
try {
|
|
2100
2237
|
const exitCode = await new Promise((resolvePromise, reject) => {
|
|
2101
2238
|
const child = spawn(args.command, args.commandArgs, {
|
|
2102
|
-
cwd: args.cwd ?
|
|
2239
|
+
cwd: args.cwd ? resolve7(args.cwd) : process.cwd(),
|
|
2103
2240
|
stdio: ["ignore", "pipe", "pipe"],
|
|
2104
2241
|
env: args.env ?? process.env,
|
|
2105
2242
|
detached: process.platform !== "win32"
|
|
@@ -2215,8 +2352,8 @@ async function runLoggedProcess(args) {
|
|
|
2215
2352
|
}
|
|
2216
2353
|
|
|
2217
2354
|
// src/local-runtime/huggingface-cache.ts
|
|
2218
|
-
import { homedir as
|
|
2219
|
-
import { dirname as dirname6, join as
|
|
2355
|
+
import { homedir as homedir3 } from "os";
|
|
2356
|
+
import { dirname as dirname6, join as join5, resolve as resolve8 } from "path";
|
|
2220
2357
|
function minimalMachineLearningEnvironment(env) {
|
|
2221
2358
|
const exact = /* @__PURE__ */ new Set([
|
|
2222
2359
|
"PATH",
|
|
@@ -2270,15 +2407,15 @@ function minimalMachineLearningEnvironment(env) {
|
|
|
2270
2407
|
return result;
|
|
2271
2408
|
}
|
|
2272
2409
|
function resolveHuggingFaceCacheLayout(modelCache) {
|
|
2273
|
-
const hfHome =
|
|
2410
|
+
const hfHome = resolve8(modelCache);
|
|
2274
2411
|
return {
|
|
2275
2412
|
hfHome,
|
|
2276
|
-
hubCache:
|
|
2413
|
+
hubCache: join5(hfHome, "hub")
|
|
2277
2414
|
};
|
|
2278
2415
|
}
|
|
2279
2416
|
function withHuggingFaceCacheEnvironment(env, modelCache) {
|
|
2280
2417
|
const inheritedHub = env.HF_HUB_CACHE ?? env.HUGGINGFACE_HUB_CACHE;
|
|
2281
|
-
const canonicalHome = modelCache ?? env.HF_HOME ?? (inheritedHub ? dirname6(
|
|
2418
|
+
const canonicalHome = modelCache ?? env.HF_HOME ?? (inheritedHub ? dirname6(resolve8(inheritedHub)) : join5(homedir3(), ".cache", "huggingface"));
|
|
2282
2419
|
const layout = resolveHuggingFaceCacheLayout(canonicalHome);
|
|
2283
2420
|
const result = {
|
|
2284
2421
|
...env,
|
|
@@ -2366,13 +2503,13 @@ async function certifiedFileDigest(path, expected) {
|
|
|
2366
2503
|
}
|
|
2367
2504
|
function isStrictPhysicalDescendant(parent, child) {
|
|
2368
2505
|
const candidate = relative2(parent, child);
|
|
2369
|
-
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep3}`) && !
|
|
2506
|
+
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep3}`) && !isAbsolute4(candidate);
|
|
2370
2507
|
}
|
|
2371
2508
|
async function verifySnapshotCacheLocation(path, modelId, revision, modelCache) {
|
|
2372
2509
|
const repository = `models--${modelId.replaceAll("/", "--")}`;
|
|
2373
2510
|
const layout = resolveHuggingFaceCacheLayout(modelCache);
|
|
2374
|
-
const repositoryRoot =
|
|
2375
|
-
const expectedPath =
|
|
2511
|
+
const repositoryRoot = join6(layout.hubCache, repository);
|
|
2512
|
+
const expectedPath = join6(repositoryRoot, "snapshots", revision);
|
|
2376
2513
|
const [
|
|
2377
2514
|
physicalCacheRoot,
|
|
2378
2515
|
physicalHubRoot,
|
|
@@ -2384,7 +2521,7 @@ async function verifySnapshotCacheLocation(path, modelId, revision, modelCache)
|
|
|
2384
2521
|
realpath2(layout.hfHome).catch(() => null),
|
|
2385
2522
|
realpath2(layout.hubCache).catch(() => null),
|
|
2386
2523
|
realpath2(repositoryRoot).catch(() => null),
|
|
2387
|
-
realpath2(
|
|
2524
|
+
realpath2(join6(repositoryRoot, "blobs")).catch(() => null),
|
|
2388
2525
|
realpath2(path).catch(() => null),
|
|
2389
2526
|
realpath2(expectedPath).catch(() => null)
|
|
2390
2527
|
]);
|
|
@@ -2400,7 +2537,7 @@ async function verifySnapshotCacheLocation(path, modelId, revision, modelCache)
|
|
|
2400
2537
|
throw new Error(`Hugging Face repository escapes its configured hub cache: ${repositoryRoot}`);
|
|
2401
2538
|
}
|
|
2402
2539
|
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalBlobRoot)) {
|
|
2403
|
-
throw new Error(`Hugging Face blob store escapes its repository: ${
|
|
2540
|
+
throw new Error(`Hugging Face blob store escapes its repository: ${join6(repositoryRoot, "blobs")}`);
|
|
2404
2541
|
}
|
|
2405
2542
|
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalExpectedPath)) {
|
|
2406
2543
|
throw new Error(`Hugging Face snapshot escapes its configured repository: ${expectedPath}`);
|
|
@@ -2449,7 +2586,7 @@ async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelC
|
|
|
2449
2586
|
);
|
|
2450
2587
|
const physicalFiles = /* @__PURE__ */ new Map();
|
|
2451
2588
|
for (const file of snapshotFiles) {
|
|
2452
|
-
const metadata = await
|
|
2589
|
+
const metadata = await lstat4(file);
|
|
2453
2590
|
const target = await realpath2(file);
|
|
2454
2591
|
if (metadata.isSymbolicLink()) {
|
|
2455
2592
|
if (!isStrictPhysicalDescendant(physicalBlobRoot, target)) {
|
|
@@ -2462,13 +2599,13 @@ async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelC
|
|
|
2462
2599
|
const expectedFiles = new Set(Object.keys(digests));
|
|
2463
2600
|
const unexpected = [...actualFiles].filter((name) => !expectedFiles.has(name)).sort();
|
|
2464
2601
|
if (unexpected.length > 0) {
|
|
2465
|
-
throw new Error(`Certified snapshot contains unexpected file: ${
|
|
2602
|
+
throw new Error(`Certified snapshot contains unexpected file: ${join6(path, unexpected[0])}`);
|
|
2466
2603
|
}
|
|
2467
2604
|
for (const [name, expected] of Object.entries(digests)) {
|
|
2468
2605
|
if (!actualFiles.has(name)) {
|
|
2469
|
-
throw new Error(`Certified snapshot file is missing: ${
|
|
2606
|
+
throw new Error(`Certified snapshot file is missing: ${join6(path, name)}`);
|
|
2470
2607
|
}
|
|
2471
|
-
const file =
|
|
2608
|
+
const file = join6(path, name);
|
|
2472
2609
|
const physicalFile = physicalFiles.get(file);
|
|
2473
2610
|
if (await certifiedFileDigest(physicalFile, expected) !== expected.digest) {
|
|
2474
2611
|
throw new Error(`Certified snapshot file checksum mismatch: ${file}`);
|
|
@@ -2487,7 +2624,7 @@ function buildModelPrefetchPayload(request, config) {
|
|
|
2487
2624
|
return {
|
|
2488
2625
|
base_model: model.id,
|
|
2489
2626
|
...revision ? { revision } : {},
|
|
2490
|
-
...config.paths.modelCache ? { model_cache:
|
|
2627
|
+
...config.paths.modelCache ? { model_cache: resolve9(config.paths.modelCache) } : {}
|
|
2491
2628
|
};
|
|
2492
2629
|
}
|
|
2493
2630
|
function safeName(value) {
|
|
@@ -2498,7 +2635,7 @@ function transformersWeightName(name) {
|
|
|
2498
2635
|
return lower.endsWith(".safetensors") && !lower.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(lower);
|
|
2499
2636
|
}
|
|
2500
2637
|
async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
2501
|
-
const root = await
|
|
2638
|
+
const root = await lstat4(path).catch(() => null);
|
|
2502
2639
|
if (!root) throw new Error(`paths.baseModel is set to ${path}, but that path does not exist.`);
|
|
2503
2640
|
if (root.isSymbolicLink()) {
|
|
2504
2641
|
throw new Error(`paths.baseModel must not itself be a symbolic link: ${path}`);
|
|
@@ -2511,7 +2648,7 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2511
2648
|
const files = [];
|
|
2512
2649
|
const visit = async (directory) => {
|
|
2513
2650
|
for (const entry of await readdir4(directory, { withFileTypes: true })) {
|
|
2514
|
-
const child =
|
|
2651
|
+
const child = join6(directory, entry.name);
|
|
2515
2652
|
if (entry.isSymbolicLink()) {
|
|
2516
2653
|
const metadata2 = await stat4(child);
|
|
2517
2654
|
if (!metadata2.isFile()) {
|
|
@@ -2535,15 +2672,15 @@ async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
|
2535
2672
|
throw new Error(`Local base-model directory contains no non-empty Transformers model weights: ${path}`);
|
|
2536
2673
|
}
|
|
2537
2674
|
const required = async (name) => {
|
|
2538
|
-
const metadata = await stat4(
|
|
2675
|
+
const metadata = await stat4(join6(path, name)).catch(() => null);
|
|
2539
2676
|
return Boolean(metadata?.isFile() && metadata.size > 0);
|
|
2540
2677
|
};
|
|
2541
2678
|
if (!await required("config.json")) throw new Error(`Local base-model directory is missing config.json: ${path}`);
|
|
2542
2679
|
try {
|
|
2543
|
-
const config = JSON.parse(await readFile5(
|
|
2680
|
+
const config = JSON.parse(await readFile5(join6(path, "config.json"), "utf8"));
|
|
2544
2681
|
assertCertifiedBaseModelConfig(
|
|
2545
2682
|
config,
|
|
2546
|
-
`Local base-model config ${
|
|
2683
|
+
`Local base-model config ${join6(path, "config.json")}`,
|
|
2547
2684
|
expectedModelId
|
|
2548
2685
|
);
|
|
2549
2686
|
} catch (error) {
|
|
@@ -2593,14 +2730,14 @@ async function prefetchBaseModel(args) {
|
|
|
2593
2730
|
...buildModelPrefetchPayload(args.request, args.config),
|
|
2594
2731
|
...args.localOnly ? { local_files_only: true } : {}
|
|
2595
2732
|
};
|
|
2596
|
-
const artifactDir =
|
|
2733
|
+
const artifactDir = resolve9(
|
|
2597
2734
|
args.config.artifactRoot,
|
|
2598
2735
|
args.localOnly ? "verify-base" : "prefetch",
|
|
2599
2736
|
`${safeName(args.request.spec_snapshot.base_model)}-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
|
|
2600
2737
|
);
|
|
2601
2738
|
await mkdir5(artifactDir, { recursive: true });
|
|
2602
2739
|
if (args.config.paths.baseModel) {
|
|
2603
|
-
const localPath =
|
|
2740
|
+
const localPath = resolve9(args.config.paths.baseModel);
|
|
2604
2741
|
const verified = await verifyLocalBaseModel(
|
|
2605
2742
|
localPath,
|
|
2606
2743
|
args.request.spec_snapshot.base_model,
|
|
@@ -2617,9 +2754,9 @@ async function prefetchBaseModel(args) {
|
|
|
2617
2754
|
reason: args.localOnly ? "Verified the configured local base-model artifact; no network access was used." : "paths.baseModel points at a verified local base-model artifact; no Hugging Face download is needed."
|
|
2618
2755
|
};
|
|
2619
2756
|
}
|
|
2620
|
-
const inputPath =
|
|
2621
|
-
const outputPath =
|
|
2622
|
-
const logPath =
|
|
2757
|
+
const inputPath = join6(artifactDir, "prefetch-input.json");
|
|
2758
|
+
const outputPath = join6(artifactDir, "prefetch-output.json");
|
|
2759
|
+
const logPath = join6(artifactDir, "prefetch.log");
|
|
2623
2760
|
await writeJson(inputPath, payload);
|
|
2624
2761
|
const entrypoint = buildBundledPythonCommand(
|
|
2625
2762
|
"prefetch.py",
|
|
@@ -2739,17 +2876,17 @@ function evaluateGeneralRegressionGate(comparison, policy) {
|
|
|
2739
2876
|
}
|
|
2740
2877
|
|
|
2741
2878
|
// src/local-runtime/orchestrator.ts
|
|
2742
|
-
import { lstat as
|
|
2879
|
+
import { lstat as lstat5, mkdir as mkdir8, open as open2, readFile as readFile8, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
|
|
2743
2880
|
import { createReadStream as createReadStream3 } from "fs";
|
|
2744
2881
|
import { createHash as createHash6, randomUUID as randomUUID4 } from "crypto";
|
|
2745
2882
|
import { performance } from "perf_hooks";
|
|
2746
|
-
import { homedir as
|
|
2747
|
-
import { basename, dirname as dirname8, join as
|
|
2883
|
+
import { homedir as homedir4 } from "os";
|
|
2884
|
+
import { basename, dirname as dirname8, join as join9, relative as relative3, resolve as resolve12 } from "path";
|
|
2748
2885
|
|
|
2749
2886
|
// src/local-runtime/evaluation.ts
|
|
2750
2887
|
import { createHash as createHash5 } from "crypto";
|
|
2751
2888
|
import { mkdir as mkdir6, readFile as readFile6, rm as rm2, writeFile as writeFile4 } from "fs/promises";
|
|
2752
|
-
import { dirname as dirname7, join as
|
|
2889
|
+
import { dirname as dirname7, join as join7, resolve as resolve10 } from "path";
|
|
2753
2890
|
function normalize(value) {
|
|
2754
2891
|
return value.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2755
2892
|
}
|
|
@@ -2984,7 +3121,7 @@ async function runTransformersInference(args) {
|
|
|
2984
3121
|
id: String(index),
|
|
2985
3122
|
input: example.input
|
|
2986
3123
|
})),
|
|
2987
|
-
model_cache: args.config.paths.modelCache ?
|
|
3124
|
+
model_cache: args.config.paths.modelCache ? resolve10(args.config.paths.modelCache) : void 0,
|
|
2988
3125
|
trust_remote_code: false,
|
|
2989
3126
|
device: args.config.evaluation.inference.device,
|
|
2990
3127
|
generation: {
|
|
@@ -3069,11 +3206,14 @@ function baselineCacheKey(args) {
|
|
|
3069
3206
|
function baselineInputsAreStable(args) {
|
|
3070
3207
|
return args.config.paths.baseModel ? Boolean(args.sourceFingerprint) : Boolean(args.baseModelRevision);
|
|
3071
3208
|
}
|
|
3209
|
+
function baselineStoreRoot(config) {
|
|
3210
|
+
return config.storeRoot ? resolve10(config.storeRoot) : defaultLocalHome();
|
|
3211
|
+
}
|
|
3072
3212
|
function baselineCachePath(config, key) {
|
|
3073
|
-
|
|
3074
|
-
return join6(root, "cache", "baseline-evals", `${key}.json`);
|
|
3213
|
+
return join7(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
|
|
3075
3214
|
}
|
|
3076
|
-
async function readBaselineCache(path) {
|
|
3215
|
+
async function readBaselineCache(config, path) {
|
|
3216
|
+
await hardenExistingLocalStore(baselineStoreRoot(config));
|
|
3077
3217
|
try {
|
|
3078
3218
|
return evalReportSchema.parse(JSON.parse(await readFile6(path, "utf8")));
|
|
3079
3219
|
} catch {
|
|
@@ -3084,7 +3224,7 @@ var cachedPackageVersion = null;
|
|
|
3084
3224
|
async function packageVersion() {
|
|
3085
3225
|
if (cachedPackageVersion !== null) return cachedPackageVersion;
|
|
3086
3226
|
try {
|
|
3087
|
-
const raw = await readFile6(
|
|
3227
|
+
const raw = await readFile6(resolve10(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
|
|
3088
3228
|
cachedPackageVersion = String(
|
|
3089
3229
|
JSON.parse(raw).version ?? "unknown"
|
|
3090
3230
|
);
|
|
@@ -3153,7 +3293,7 @@ async function evaluateExamples(args) {
|
|
|
3153
3293
|
baseModelId,
|
|
3154
3294
|
args.baseModelRevision
|
|
3155
3295
|
);
|
|
3156
|
-
const localModelSource = args.modelSource ?
|
|
3296
|
+
const localModelSource = args.modelSource ? resolve10(args.modelSource) !== resolve10(baseModelId) : false;
|
|
3157
3297
|
const baseModelRevision = localModelSource ? defaultBaseModelRevision(baseModelId) : requestedBaseModelRevision;
|
|
3158
3298
|
const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
|
|
3159
3299
|
const truncated = args.examples.length > maxExamples;
|
|
@@ -3178,7 +3318,7 @@ async function evaluateExamples(args) {
|
|
|
3178
3318
|
packageVersion: await packageVersion()
|
|
3179
3319
|
}) : null;
|
|
3180
3320
|
if (cacheKey) {
|
|
3181
|
-
const cached = await readBaselineCache(baselineCachePath(args.config, cacheKey));
|
|
3321
|
+
const cached = await readBaselineCache(args.config, baselineCachePath(args.config, cacheKey));
|
|
3182
3322
|
if (cached) {
|
|
3183
3323
|
const report2 = evalReportSchema.parse({
|
|
3184
3324
|
...cached,
|
|
@@ -3229,7 +3369,8 @@ async function evaluateExamples(args) {
|
|
|
3229
3369
|
logUri: inference ? fileUri(`${args.outputPath}.inference.log`) : void 0
|
|
3230
3370
|
});
|
|
3231
3371
|
if (cacheKey) {
|
|
3232
|
-
await
|
|
3372
|
+
await createLocalStore(baselineStoreRoot(args.config)).ensure();
|
|
3373
|
+
await writePrivateJsonAtomic(
|
|
3233
3374
|
baselineCachePath(args.config, cacheKey),
|
|
3234
3375
|
{ ...report, cache_key: cacheKey }
|
|
3235
3376
|
);
|
|
@@ -3269,7 +3410,7 @@ function compareEvalReports(baseline, candidate) {
|
|
|
3269
3410
|
|
|
3270
3411
|
// src/local-runtime/process-training.ts
|
|
3271
3412
|
import { access, mkdir as mkdir7, readFile as readFile7, writeFile as writeFile5 } from "fs/promises";
|
|
3272
|
-
import { join as
|
|
3413
|
+
import { join as join8, resolve as resolve11 } from "path";
|
|
3273
3414
|
function buildTrainingHyperparameters(request, options = {}) {
|
|
3274
3415
|
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
3275
3416
|
const hyper = request.hyperparameters;
|
|
@@ -3365,7 +3506,7 @@ async function launchProcessTraining(args) {
|
|
|
3365
3506
|
const hyperparameters = buildTrainingHyperparameters(request, {
|
|
3366
3507
|
baseModelRevision: args.baseModelRevision
|
|
3367
3508
|
});
|
|
3368
|
-
await writeJson(
|
|
3509
|
+
await writeJson(join8(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
|
|
3369
3510
|
const entrypoint = buildBundledPythonCommand("train.py");
|
|
3370
3511
|
if (config.dryRun) {
|
|
3371
3512
|
await writeFile5(
|
|
@@ -3390,20 +3531,20 @@ async function launchProcessTraining(args) {
|
|
|
3390
3531
|
withOfflineHuggingFaceCacheEnvironment({
|
|
3391
3532
|
...inheritedEnv,
|
|
3392
3533
|
BACKEND: "local",
|
|
3393
|
-
SM_CHANNEL_TRAINING:
|
|
3394
|
-
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ?
|
|
3395
|
-
SM_MODEL_DIR:
|
|
3396
|
-
SM_OUTPUT_DIR:
|
|
3397
|
-
TT_HYPERPARAMETERS_PATH:
|
|
3534
|
+
SM_CHANNEL_TRAINING: resolve11(artifacts.trainingInputDir),
|
|
3535
|
+
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve11(config.paths.baseModel) : void 0,
|
|
3536
|
+
SM_MODEL_DIR: resolve11(artifacts.trainingModelDir),
|
|
3537
|
+
SM_OUTPUT_DIR: resolve11(artifacts.trainingOutputDir),
|
|
3538
|
+
TT_HYPERPARAMETERS_PATH: resolve11(artifacts.trainingConfigDir, "hyperparameters.json")
|
|
3398
3539
|
}, config.paths.modelCache)
|
|
3399
3540
|
);
|
|
3400
3541
|
for (const [key, value] of Object.entries(childEnv)) {
|
|
3401
3542
|
if (value === void 0) delete childEnv[key];
|
|
3402
3543
|
}
|
|
3403
|
-
if (config.paths.modelCache) await mkdir7(
|
|
3544
|
+
if (config.paths.modelCache) await mkdir7(resolve11(config.paths.modelCache), { recursive: true });
|
|
3404
3545
|
if (config.paths.baseModel) {
|
|
3405
3546
|
await verifyLocalBaseModel(
|
|
3406
|
-
|
|
3547
|
+
resolve11(config.paths.baseModel),
|
|
3407
3548
|
request.spec_snapshot.base_model,
|
|
3408
3549
|
config.paths.modelCache
|
|
3409
3550
|
);
|
|
@@ -3430,12 +3571,12 @@ async function launchProcessTraining(args) {
|
|
|
3430
3571
|
onLine: (line) => forwardTrainingProgress(line),
|
|
3431
3572
|
shouldCancel: args.shouldCancel
|
|
3432
3573
|
});
|
|
3433
|
-
const metricsPath =
|
|
3574
|
+
const metricsPath = join8(artifacts.trainingModelDir, "training-metrics.json");
|
|
3434
3575
|
const metrics = await pathExists(metricsPath) ? JSON.parse(await readFile7(metricsPath, "utf8")) : null;
|
|
3435
3576
|
if (exitCode !== 0) {
|
|
3436
3577
|
throw new Error(`Training process exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
|
|
3437
3578
|
}
|
|
3438
|
-
const modelTarPath =
|
|
3579
|
+
const modelTarPath = join8(artifacts.trainingOutputDir, "model.tar.gz");
|
|
3439
3580
|
const modelUri = await pathExists(modelTarPath) ? fileUri(modelTarPath) : fileUri(artifacts.trainingModelDir);
|
|
3440
3581
|
await args.reporter?.onEvent?.({
|
|
3441
3582
|
stage: "training",
|
|
@@ -3537,14 +3678,14 @@ async function loadRunRequest(path) {
|
|
|
3537
3678
|
}
|
|
3538
3679
|
async function loadLocalRunnerConfig(path) {
|
|
3539
3680
|
if (!path) return localRunnerConfigSchema.parse({});
|
|
3540
|
-
const configPath =
|
|
3681
|
+
const configPath = resolve12(path);
|
|
3541
3682
|
const base = dirname8(configPath);
|
|
3542
3683
|
const config = localRunnerConfigSchema.parse(await loadJsonFile(configPath));
|
|
3543
3684
|
const configPathValue = (value) => {
|
|
3544
3685
|
if (!value) return void 0;
|
|
3545
|
-
if (value === "~") return
|
|
3546
|
-
if (value.startsWith("~/")) return
|
|
3547
|
-
return
|
|
3686
|
+
if (value === "~") return homedir4();
|
|
3687
|
+
if (value.startsWith("~/")) return resolve12(homedir4(), value.slice(2));
|
|
3688
|
+
return resolve12(base, value);
|
|
3548
3689
|
};
|
|
3549
3690
|
return {
|
|
3550
3691
|
...config,
|
|
@@ -3742,21 +3883,21 @@ async function hashFileIfPresent(path) {
|
|
|
3742
3883
|
}
|
|
3743
3884
|
async function packageVersion2() {
|
|
3744
3885
|
try {
|
|
3745
|
-
const parsed = JSON.parse(await readFile8(
|
|
3886
|
+
const parsed = JSON.parse(await readFile8(resolve12(packageRoot2, "package.json"), "utf8"));
|
|
3746
3887
|
return typeof parsed.version === "string" ? parsed.version : "unknown";
|
|
3747
3888
|
} catch {
|
|
3748
3889
|
return "unknown";
|
|
3749
3890
|
}
|
|
3750
3891
|
}
|
|
3751
3892
|
async function runtimeFingerprint() {
|
|
3752
|
-
const bundledProject2 =
|
|
3893
|
+
const bundledProject2 = resolve12(packageRoot2, "training/adapter");
|
|
3753
3894
|
return hashJson({
|
|
3754
3895
|
tt_local_version: await packageVersion2(),
|
|
3755
3896
|
node_version: process.version,
|
|
3756
3897
|
platform: process.platform,
|
|
3757
3898
|
architecture: process.arch,
|
|
3758
|
-
runner_pyproject: await hashFileIfPresent(
|
|
3759
|
-
runner_uv_lock: await hashFileIfPresent(
|
|
3899
|
+
runner_pyproject: await hashFileIfPresent(resolve12(bundledProject2, "pyproject.toml")),
|
|
3900
|
+
runner_uv_lock: await hashFileIfPresent(resolve12(bundledProject2, "uv.lock"))
|
|
3760
3901
|
});
|
|
3761
3902
|
}
|
|
3762
3903
|
function preparedSourceFingerprint(args) {
|
|
@@ -3803,7 +3944,7 @@ async function resolveBaseModelRevision(request, config) {
|
|
|
3803
3944
|
}
|
|
3804
3945
|
const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
|
|
3805
3946
|
const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
|
|
3806
|
-
const refPath =
|
|
3947
|
+
const refPath = resolve12(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
|
|
3807
3948
|
try {
|
|
3808
3949
|
const revision = (await readFile8(refPath, "utf8")).trim();
|
|
3809
3950
|
if (revision) return revision;
|
|
@@ -3813,7 +3954,7 @@ async function resolveBaseModelRevision(request, config) {
|
|
|
3813
3954
|
}
|
|
3814
3955
|
async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
|
|
3815
3956
|
const root = localModelArtifactPath(uri);
|
|
3816
|
-
const rootMetadata = await
|
|
3957
|
+
const rootMetadata = await lstat5(root);
|
|
3817
3958
|
if (rootMetadata.isSymbolicLink()) {
|
|
3818
3959
|
throw new Error(`Local base model path must not itself be a symbolic link: ${root}`);
|
|
3819
3960
|
}
|
|
@@ -3824,7 +3965,7 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
|
|
|
3824
3965
|
const files = [];
|
|
3825
3966
|
const visit = async (directory) => {
|
|
3826
3967
|
for (const entry of await readdir5(directory, { withFileTypes: true })) {
|
|
3827
|
-
const child =
|
|
3968
|
+
const child = resolve12(directory, entry.name);
|
|
3828
3969
|
if (entry.isDirectory()) {
|
|
3829
3970
|
await visit(child);
|
|
3830
3971
|
continue;
|
|
@@ -3868,7 +4009,7 @@ async function removePrefixedArtifacts(path) {
|
|
|
3868
4009
|
const names = await readdir5(directory).catch(() => []);
|
|
3869
4010
|
await Promise.all([
|
|
3870
4011
|
rm3(path, { force: true }),
|
|
3871
|
-
...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(
|
|
4012
|
+
...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(resolve12(directory, name), { recursive: true, force: true }))
|
|
3872
4013
|
]);
|
|
3873
4014
|
}
|
|
3874
4015
|
async function cleanupStageArtifacts(artifacts, stage) {
|
|
@@ -3881,7 +4022,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
3881
4022
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
3882
4023
|
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
3883
4024
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
3884
|
-
rm3(
|
|
4025
|
+
rm3(resolve12(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
3885
4026
|
removeReport(),
|
|
3886
4027
|
rm3(artifacts.artifactManifestJson, { force: true })
|
|
3887
4028
|
]);
|
|
@@ -3899,7 +4040,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
3899
4040
|
await Promise.all([
|
|
3900
4041
|
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
3901
4042
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
3902
|
-
rm3(
|
|
4043
|
+
rm3(resolve12(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
3903
4044
|
removePrefixedArtifacts(artifacts.candidateEvalJson),
|
|
3904
4045
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
3905
4046
|
removeReport()
|
|
@@ -3989,11 +4130,12 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
3989
4130
|
throw new Error(`Unable to acquire local workflow lock for ${description}.`);
|
|
3990
4131
|
}
|
|
3991
4132
|
async function acquireRunLock(store, runId) {
|
|
3992
|
-
|
|
4133
|
+
await store.ensure();
|
|
4134
|
+
return acquireWorkflowLock(resolve12(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
|
|
3993
4135
|
}
|
|
3994
4136
|
async function acquireArtifactLock(artifacts) {
|
|
3995
4137
|
return acquireWorkflowLock(
|
|
3996
|
-
|
|
4138
|
+
join9(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
|
|
3997
4139
|
`Artifact directory ${artifacts.runDir}`
|
|
3998
4140
|
);
|
|
3999
4141
|
}
|
|
@@ -4040,13 +4182,13 @@ function stageFingerprintPath(prepared, stage) {
|
|
|
4040
4182
|
return `${stageOutputPath(prepared, stage)}.stage.json`;
|
|
4041
4183
|
}
|
|
4042
4184
|
async function runnerFileFingerprints() {
|
|
4043
|
-
const project =
|
|
4044
|
-
const bundledSource =
|
|
4045
|
-
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) =>
|
|
4185
|
+
const project = resolve12(packageRoot2, "training/adapter");
|
|
4186
|
+
const bundledSource = resolve12(packageRoot2, "training/adapter/src");
|
|
4187
|
+
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve12(bundledSource, entry.name));
|
|
4046
4188
|
const candidates = [.../* @__PURE__ */ new Set([
|
|
4047
4189
|
...pythonSources,
|
|
4048
|
-
|
|
4049
|
-
|
|
4190
|
+
resolve12(project, "pyproject.toml"),
|
|
4191
|
+
resolve12(project, "uv.lock")
|
|
4050
4192
|
])];
|
|
4051
4193
|
const fingerprints = {};
|
|
4052
4194
|
for (const path of candidates.sort()) fingerprints[path] = await hashFileIfPresent(path);
|
|
@@ -4160,7 +4302,7 @@ async function ensureRunRecord(args) {
|
|
|
4160
4302
|
`Run ${args.request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
|
|
4161
4303
|
);
|
|
4162
4304
|
}
|
|
4163
|
-
if (
|
|
4305
|
+
if (resolve12(existing.artifact_dir) !== resolve12(args.artifacts.runDir)) {
|
|
4164
4306
|
throw new Error(
|
|
4165
4307
|
`Run ${args.request.run_id} is already bound to artifact directory ${existing.artifact_dir}; it cannot be resumed with ${args.artifacts.runDir}.`
|
|
4166
4308
|
);
|
|
@@ -4267,7 +4409,7 @@ async function prepareStage(args) {
|
|
|
4267
4409
|
const preparedExists = await pathExists2(args.artifacts.stageMetadataJson) && await pathExists2(args.artifacts.trainingJsonl);
|
|
4268
4410
|
const prepared = await computePreparedRun({ ...args, writeArtifacts: false });
|
|
4269
4411
|
await args.store.syncRunRequest(args.request, args.artifacts.runDir);
|
|
4270
|
-
await writeJsonAtomic(
|
|
4412
|
+
await writeJsonAtomic(resolve12(args.artifacts.runDir, "request.json"), args.request);
|
|
4271
4413
|
const existingMetadata = preparedExists ? await readStageMetadata(args.artifacts.stageMetadataJson) : null;
|
|
4272
4414
|
let canReuse = preparedExists && existingMetadata?.source_fingerprint === prepared.metadata.source_fingerprint;
|
|
4273
4415
|
if (canReuse) {
|
|
@@ -4796,7 +4938,7 @@ async function runLocalPipeline(input) {
|
|
|
4796
4938
|
if (step.uses === "evaluate") {
|
|
4797
4939
|
const isBase = step.with.model === "base";
|
|
4798
4940
|
const report = isBase ? await runBaselineStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter }) : await runCandidateStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
|
|
4799
|
-
const reportPath =
|
|
4941
|
+
const reportPath = join9(artifacts.runDir, "evaluations", `${step.id}.json`);
|
|
4800
4942
|
await writeJsonAtomic(reportPath, { ...report, artifact_uri: fileUri(reportPath) });
|
|
4801
4943
|
outputs[step.id] = { report: { ...report, artifact_uri: fileUri(reportPath) }, reportPath };
|
|
4802
4944
|
await refreshArtifactManifest(prepared);
|
|
@@ -4872,7 +5014,6 @@ async function runLocalFineTune(input) {
|
|
|
4872
5014
|
}
|
|
4873
5015
|
|
|
4874
5016
|
export {
|
|
4875
|
-
writeJsonAtomic,
|
|
4876
5017
|
assertArtifactManifest,
|
|
4877
5018
|
TRAINING_MODELS,
|
|
4878
5019
|
resolveTrainingModel,
|
|
@@ -4880,6 +5021,18 @@ export {
|
|
|
4880
5021
|
defaultBaseModelRevision,
|
|
4881
5022
|
resolveModelLoader,
|
|
4882
5023
|
assertUsableModelArtifact,
|
|
5024
|
+
DEFAULT_ARTIFACT_ROOT,
|
|
5025
|
+
DEFAULT_FOUNDATION_RUNS_DIR,
|
|
5026
|
+
expandUserPath,
|
|
5027
|
+
getConfigDir,
|
|
5028
|
+
getAgentConfigDir,
|
|
5029
|
+
defaultStoreRoot,
|
|
5030
|
+
isTerminalRunState,
|
|
5031
|
+
defaultLocalHome,
|
|
5032
|
+
localStorePaths,
|
|
5033
|
+
hardenExistingLocalStore,
|
|
5034
|
+
writePrivateJsonAtomic,
|
|
5035
|
+
createLocalStore,
|
|
4883
5036
|
datasetFormatSchema,
|
|
4884
5037
|
baseModelRevisionSchema,
|
|
4885
5038
|
behaviorSpecExampleSchema,
|
|
@@ -4926,10 +5079,6 @@ export {
|
|
|
4926
5079
|
buildFoundationPythonCommand,
|
|
4927
5080
|
withFoundationPythonEnvironment,
|
|
4928
5081
|
runLoggedProcess,
|
|
4929
|
-
isTerminalRunState,
|
|
4930
|
-
defaultLocalHome,
|
|
4931
|
-
localStorePaths,
|
|
4932
|
-
createLocalStore,
|
|
4933
5082
|
minimalMachineLearningEnvironment,
|
|
4934
5083
|
withHuggingFaceCacheEnvironment,
|
|
4935
5084
|
withOfflineHuggingFaceCacheEnvironment,
|
|
@@ -4949,4 +5098,4 @@ export {
|
|
|
4949
5098
|
runLocalPipeline,
|
|
4950
5099
|
runLocalFineTune
|
|
4951
5100
|
};
|
|
4952
|
-
//# sourceMappingURL=chunk-
|
|
5101
|
+
//# sourceMappingURL=chunk-7VIUHQBI.js.map
|