@tuned-tensor/cli 0.15.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +124 -46
- package/dist/{chunk-VVKC563R.js → chunk-KRKISPHG.js} +420 -98
- package/dist/chunk-KRKISPHG.js.map +1 -0
- package/dist/index.js +1070 -706
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +147 -32
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/architecture.md +17 -6
- package/docs/local-runtime/aws-gpu.md +248 -0
- package/docs/local-runtime/foundation-long-runs.md +21 -5
- package/docs/local-runtime/serving.md +11 -1
- package/docs/spec-architecture.html +573 -0
- package/docs/spec-workflow.md +110 -0
- package/examples/local-runtime/aws-runner.json +13 -0
- package/package.json +6 -4
- package/training/adapter/src/foundation_export.py +81 -0
- package/training/adapter/src/serve.py +12 -5
- package/training/adapter/src/train.py +4 -0
- package/training/foundation/pyproject.toml +4 -0
- package/training/foundation/src/model.py +2 -0
- package/training/foundation/uv.lock +225 -0
- package/dist/chunk-VVKC563R.js.map +0 -1
|
@@ -1076,7 +1076,7 @@ var comparisonReportSchema = z.object({
|
|
|
1076
1076
|
}).strict())
|
|
1077
1077
|
}).strict();
|
|
1078
1078
|
var trainingReportSchema = z.object({
|
|
1079
|
-
provider: z.
|
|
1079
|
+
provider: z.enum(["local-uv", "aws-ssh"]),
|
|
1080
1080
|
training_job_name: z.string(),
|
|
1081
1081
|
model_artifact_uri: z.string().optional(),
|
|
1082
1082
|
base_model_artifact_uri: z.string().optional(),
|
|
@@ -1167,6 +1167,16 @@ var evaluationConfigSchema = z.object({
|
|
|
1167
1167
|
}).strict().optional()
|
|
1168
1168
|
}).strict();
|
|
1169
1169
|
var localRunnerConfigSchema = z.object({
|
|
1170
|
+
gpu: z.object({
|
|
1171
|
+
provider: z.literal("aws"),
|
|
1172
|
+
instanceId: z.string().regex(/^i-[a-f0-9]{8,17}$/),
|
|
1173
|
+
profile: z.string().min(1).optional(),
|
|
1174
|
+
region: z.string().min(1).optional(),
|
|
1175
|
+
user: z.string().regex(/^[a-z_][a-z0-9_-]*$/i),
|
|
1176
|
+
identityFile: z.string().min(1).optional(),
|
|
1177
|
+
privateIp: z.boolean().default(false),
|
|
1178
|
+
maxSeconds: z.number().int().min(60).max(604800).default(86400)
|
|
1179
|
+
}).strict().optional(),
|
|
1170
1180
|
storeRoot: z.string().optional(),
|
|
1171
1181
|
artifactRoot: z.string().default(DEFAULT_ARTIFACT_ROOT),
|
|
1172
1182
|
dryRun: z.boolean().default(false),
|
|
@@ -1428,6 +1438,56 @@ async function loadLocalRunInput(path, options = {}) {
|
|
|
1428
1438
|
options
|
|
1429
1439
|
);
|
|
1430
1440
|
}
|
|
1441
|
+
function specHash(source) {
|
|
1442
|
+
return createHash2("sha256").update(source).digest("hex");
|
|
1443
|
+
}
|
|
1444
|
+
function validateBehaviorSpec(input) {
|
|
1445
|
+
const errors = [];
|
|
1446
|
+
const warnings = [];
|
|
1447
|
+
if (input.kind === "request") {
|
|
1448
|
+
return { valid: false, errors: ["Use a behavior spec, not a run request."], warnings };
|
|
1449
|
+
}
|
|
1450
|
+
const { spec } = input;
|
|
1451
|
+
const placeholders = input.kind === "foundation-spec" ? foundationPlaceholderIssues(input.spec) : generatedPlaceholderIssues(input.request);
|
|
1452
|
+
if (placeholders.length) {
|
|
1453
|
+
errors.push(`Edit the generated behavior spec before training: ${placeholders.join("; ")}.`);
|
|
1454
|
+
}
|
|
1455
|
+
if (input.kind === "foundation-spec") {
|
|
1456
|
+
if (input.spec.foundation.nproc_per_node !== 1) {
|
|
1457
|
+
errors.push("Foundation nproc_per_node must be 1 for this runtime.");
|
|
1458
|
+
}
|
|
1459
|
+
const nonNumericOutput = spec.examples.some(
|
|
1460
|
+
(example) => !/-?\d+(?:\.\d+)?/.test(example.output.replaceAll(",", ""))
|
|
1461
|
+
);
|
|
1462
|
+
if (input.spec.foundation.rl_steps > 0 && nonNumericOutput) {
|
|
1463
|
+
errors.push("Foundation RL requires numeric expected outputs in every example.");
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
for (const key of ["name", "system_prompt"]) {
|
|
1467
|
+
if (!spec[key].trim()) errors.push(`${key} must not be blank.`);
|
|
1468
|
+
}
|
|
1469
|
+
if (spec.examples.length < 2 && !(input.kind === "spec" && input.spec.dataset_prebuilt)) {
|
|
1470
|
+
errors.push("Add at least two examples for training and evaluation.");
|
|
1471
|
+
}
|
|
1472
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1473
|
+
for (const [index, example] of spec.examples.entries()) {
|
|
1474
|
+
if (!example.input.trim() || !example.output.trim()) {
|
|
1475
|
+
errors.push(`examples[${index}] input and output must not be blank.`);
|
|
1476
|
+
}
|
|
1477
|
+
const key = example.input.trim();
|
|
1478
|
+
if (seen.has(key)) {
|
|
1479
|
+
errors.push(`examples[${index}] repeats an input; use distinct examples to avoid conflicting labels or train/eval overlap.`);
|
|
1480
|
+
}
|
|
1481
|
+
seen.add(key);
|
|
1482
|
+
}
|
|
1483
|
+
if (!spec.guidelines.length) {
|
|
1484
|
+
warnings.push("Add guidelines to make expected behavior explicit.");
|
|
1485
|
+
}
|
|
1486
|
+
if (spec.examples.length < 10) {
|
|
1487
|
+
warnings.push("This is a small example set; passing validation does not establish model quality.");
|
|
1488
|
+
}
|
|
1489
|
+
return { valid: errors.length === 0, errors, warnings };
|
|
1490
|
+
}
|
|
1431
1491
|
|
|
1432
1492
|
// src/local-runtime/dataset.ts
|
|
1433
1493
|
import { readFile as readFile3 } from "fs/promises";
|
|
@@ -2907,17 +2967,240 @@ function evaluateGeneralRegressionGate(comparison, policy) {
|
|
|
2907
2967
|
}
|
|
2908
2968
|
|
|
2909
2969
|
// src/local-runtime/orchestrator.ts
|
|
2910
|
-
import { lstat as lstat5, mkdir as
|
|
2970
|
+
import { lstat as lstat5, mkdir as mkdir9, open as open3, readFile as readFile8, readdir as readdir5, rename as rename3, rm as rm4, stat as stat7 } from "fs/promises";
|
|
2911
2971
|
import { createReadStream as createReadStream3 } from "fs";
|
|
2912
|
-
import { createHash as
|
|
2972
|
+
import { createHash as createHash7, randomUUID as randomUUID5 } from "crypto";
|
|
2913
2973
|
import { performance } from "perf_hooks";
|
|
2914
2974
|
import { homedir as homedir4 } from "os";
|
|
2915
|
-
import { basename, dirname as
|
|
2975
|
+
import { basename as basename2, dirname as dirname9, join as join10, relative as relative4, resolve as resolve13 } from "path";
|
|
2976
|
+
|
|
2977
|
+
// src/local-runtime/gpu-executor.ts
|
|
2978
|
+
import { createHash as createHash5, randomUUID as randomUUID4 } from "crypto";
|
|
2979
|
+
import { mkdtemp, mkdir as mkdir6, realpath as realpath3, rm as rm2, stat as stat5, writeFile as writeFile4 } from "fs/promises";
|
|
2980
|
+
import { tmpdir } from "os";
|
|
2981
|
+
import { basename, dirname as dirname7, join as join7, resolve as resolve10 } from "path";
|
|
2982
|
+
function shellQuote(value) {
|
|
2983
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
2984
|
+
}
|
|
2985
|
+
async function resolveGpuHost(gpu) {
|
|
2986
|
+
let stdout = "";
|
|
2987
|
+
const result = await runLoggedProcess({
|
|
2988
|
+
command: "aws",
|
|
2989
|
+
commandArgs: [
|
|
2990
|
+
"ec2",
|
|
2991
|
+
"describe-instances",
|
|
2992
|
+
"--instance-ids",
|
|
2993
|
+
gpu.instanceId,
|
|
2994
|
+
...gpu.profile ? ["--profile", gpu.profile] : [],
|
|
2995
|
+
...gpu.region ? ["--region", gpu.region] : [],
|
|
2996
|
+
"--output",
|
|
2997
|
+
"json",
|
|
2998
|
+
"--no-cli-pager"
|
|
2999
|
+
],
|
|
3000
|
+
timeoutMs: 3e4,
|
|
3001
|
+
stage: "gpu_connect",
|
|
3002
|
+
onLine: (line, stream) => {
|
|
3003
|
+
if (stream === "stdout") stdout += line;
|
|
3004
|
+
}
|
|
3005
|
+
});
|
|
3006
|
+
if (result.exitCode !== 0) throw new Error(`AWS GPU lookup failed: ${result.stderr}`);
|
|
3007
|
+
const instances = JSON.parse(stdout).Reservations?.flatMap((r) => r.Instances ?? []);
|
|
3008
|
+
const instance = instances?.find((i) => i.InstanceId === gpu.instanceId);
|
|
3009
|
+
if (instance?.State?.Name !== "running") throw new Error(`AWS GPU instance ${gpu.instanceId} must already be running.`);
|
|
3010
|
+
const host = gpu.privateIp ? instance.PrivateIpAddress : instance.PublicIpAddress;
|
|
3011
|
+
if (typeof host !== "string" || !/^[0-9.]+$/.test(host)) {
|
|
3012
|
+
throw new Error("AWS GPU instance has no selected IP address. Use privateIp with a reachable VPC/VPN, or assign a public IP.");
|
|
3013
|
+
}
|
|
3014
|
+
return `${gpu.user}@${host}`;
|
|
3015
|
+
}
|
|
3016
|
+
function sshOptions(gpu) {
|
|
3017
|
+
return [
|
|
3018
|
+
"-o",
|
|
3019
|
+
"BatchMode=yes",
|
|
3020
|
+
"-o",
|
|
3021
|
+
"StrictHostKeyChecking=yes",
|
|
3022
|
+
"-o",
|
|
3023
|
+
"ConnectTimeout=15",
|
|
3024
|
+
"-o",
|
|
3025
|
+
"ServerAliveInterval=15",
|
|
3026
|
+
"-o",
|
|
3027
|
+
"ServerAliveCountMax=3",
|
|
3028
|
+
...gpu.identityFile ? ["-i", gpu.identityFile] : []
|
|
3029
|
+
];
|
|
3030
|
+
}
|
|
3031
|
+
async function checkAwsGpu(gpu) {
|
|
3032
|
+
const host = await resolveGpuHost(gpu);
|
|
3033
|
+
const result = await runLoggedProcess({
|
|
3034
|
+
command: "ssh",
|
|
3035
|
+
commandArgs: [
|
|
3036
|
+
...sshOptions(gpu),
|
|
3037
|
+
host,
|
|
3038
|
+
`bash -lc ${shellQuote("command -v uv && command -v rsync && command -v timeout && command -v setsid && nvidia-smi")}`
|
|
3039
|
+
],
|
|
3040
|
+
stage: "gpu_check",
|
|
3041
|
+
timeoutMs: 3e4
|
|
3042
|
+
});
|
|
3043
|
+
if (result.exitCode !== 0) throw new Error(`AWS GPU preflight failed: ${result.stderr}`);
|
|
3044
|
+
}
|
|
3045
|
+
function gpuBaseModelPath(config, model, revision) {
|
|
3046
|
+
if (config.paths.baseModel) return resolve10(config.paths.baseModel);
|
|
3047
|
+
if (!revision || !/^[a-f0-9]{40}$/i.test(revision)) {
|
|
3048
|
+
throw new Error("AWS GPU execution needs an immutable base-model revision. Run tt models prefetch first or configure paths.baseModel.");
|
|
3049
|
+
}
|
|
3050
|
+
const env = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
|
|
3051
|
+
return join7(env.HF_HUB_CACHE, `models--${model.replaceAll("/", "--")}`, "snapshots", revision);
|
|
3052
|
+
}
|
|
3053
|
+
async function physicalTransferPath(path) {
|
|
3054
|
+
try {
|
|
3055
|
+
return await realpath3(path);
|
|
3056
|
+
} catch (error) {
|
|
3057
|
+
if (error.code !== "ENOENT" || dirname7(path) === path) throw error;
|
|
3058
|
+
return join7(await physicalTransferPath(dirname7(path)), basename(path));
|
|
3059
|
+
}
|
|
3060
|
+
}
|
|
3061
|
+
async function assertSeparateTransferPaths(files) {
|
|
3062
|
+
const paths = await Promise.all(files.map((file) => physicalTransferPath(resolve10(file.path))));
|
|
3063
|
+
for (let i = 0; i < files.length; i += 1) {
|
|
3064
|
+
for (let j = 0; j < i; j += 1) {
|
|
3065
|
+
if (files[i].direction === "input" && files[j].direction === "input") continue;
|
|
3066
|
+
const left = paths[i];
|
|
3067
|
+
const right = paths[j];
|
|
3068
|
+
if (left === right || files[i].directory && right.startsWith(`${left}/`) || files[j].directory && left.startsWith(`${right}/`)) {
|
|
3069
|
+
throw new Error(`GPU transfer paths overlap: ${files[i].path} and ${files[j].path}. Use separate input, output, recovery, and backup directories.`);
|
|
3070
|
+
}
|
|
3071
|
+
}
|
|
3072
|
+
}
|
|
3073
|
+
}
|
|
3074
|
+
async function runGpuProcess(args) {
|
|
3075
|
+
if (await args.shouldCancel?.()) throw new ProcessCancelledError();
|
|
3076
|
+
await assertSeparateTransferPaths(args.files);
|
|
3077
|
+
const host = await resolveGpuHost(args.gpu);
|
|
3078
|
+
const options = sshOptions(args.gpu);
|
|
3079
|
+
const identity = args.logPath ? createHash5("sha256").update(resolve10(args.logPath)).digest("hex").slice(0, 32) : randomUUID4();
|
|
3080
|
+
const root = `/tmp/tt-gpu-${identity}`;
|
|
3081
|
+
const scratch = await mkdtemp(join7(tmpdir(), "tt-gpu-"));
|
|
3082
|
+
const runtime = join7(localRuntimePackageRoot(import.meta.url), "training", args.runtime);
|
|
3083
|
+
const mappings = args.files.map((file, index) => ({ ...file, path: resolve10(file.path), remote: `${root}/files/${index}/${basename(file.path)}` }));
|
|
3084
|
+
const mapPath = (path) => {
|
|
3085
|
+
const absolute = resolve10(path);
|
|
3086
|
+
const match = [...mappings].sort((a, b) => b.path.length - a.path.length).find((file) => absolute === file.path || file.directory && absolute.startsWith(`${file.path}/`));
|
|
3087
|
+
if (match) return match.remote + absolute.slice(match.path.length);
|
|
3088
|
+
if (absolute === runtime || absolute.startsWith(`${runtime}/`)) return `${root}/runtime${absolute.slice(runtime.length)}`;
|
|
3089
|
+
throw new Error(`GPU process path is not declared for transfer: ${path}`);
|
|
3090
|
+
};
|
|
3091
|
+
const transport = async (command, commandArgs, cancellable = true) => {
|
|
3092
|
+
const result = await runLoggedProcess({
|
|
3093
|
+
command,
|
|
3094
|
+
commandArgs,
|
|
3095
|
+
stage: args.stage,
|
|
3096
|
+
reporter: args.reporter,
|
|
3097
|
+
timeoutMs: 30 * 6e4,
|
|
3098
|
+
shouldCancel: cancellable ? args.shouldCancel : void 0
|
|
3099
|
+
});
|
|
3100
|
+
if (result.exitCode !== 0) throw new Error(`${command} failed for ${host}: ${result.stderr}`);
|
|
3101
|
+
};
|
|
3102
|
+
const ssh = async (script, cancellable = true) => transport("ssh", [...options, host, script], cancellable);
|
|
3103
|
+
const sync = async (source, destination, upload, cancellable = true, mirror = false) => {
|
|
3104
|
+
await transport("rsync", [
|
|
3105
|
+
"-rlt",
|
|
3106
|
+
"--copy-links",
|
|
3107
|
+
"--protect-args",
|
|
3108
|
+
"--modify-window=-1",
|
|
3109
|
+
"--chmod=Du=rwx,Dgo=,Fu=rw,Fgo=",
|
|
3110
|
+
...!upload ? ["--delay-updates", ...mirror ? ["--delete-delay"] : []] : [],
|
|
3111
|
+
"--exclude=.venv",
|
|
3112
|
+
"--exclude=__pycache__",
|
|
3113
|
+
"--exclude=.git",
|
|
3114
|
+
"-e",
|
|
3115
|
+
["ssh", ...options].map(shellQuote).join(" "),
|
|
3116
|
+
"--",
|
|
3117
|
+
upload ? source : `${host}:${source}`,
|
|
3118
|
+
upload ? `${host}:${destination}` : destination
|
|
3119
|
+
], cancellable);
|
|
3120
|
+
};
|
|
3121
|
+
const grace = Math.ceil((args.shutdownGraceMs ?? 5e3) / 1e3);
|
|
3122
|
+
const stop = `kill -TERM -- -"$pid" 2>/dev/null || true; remaining=${grace}; while kill -0 -- -"$pid" 2>/dev/null && [ "$remaining" -gt 0 ]; do sleep 1; remaining=$((remaining - 1)); done; kill -KILL -- -"$pid" 2>/dev/null || true`;
|
|
3123
|
+
const owner = randomUUID4();
|
|
3124
|
+
let reservationAttempted = false;
|
|
3125
|
+
let started = false;
|
|
3126
|
+
let completed = false;
|
|
3127
|
+
try {
|
|
3128
|
+
reservationAttempted = true;
|
|
3129
|
+
await ssh(`umask 077; mkdir ${root} && printf '%s' ${shellQuote(owner)} > ${root}/.owner && mkdir ${root}/runtime ${root}/files`).catch((error) => {
|
|
3130
|
+
if (error instanceof ProcessCancelledError) throw error;
|
|
3131
|
+
throw new Error(`Cannot reserve GPU staging directory ${host}:${root}. Another process or unrecovered artifacts may be present. ${error instanceof Error ? error.message : error}`);
|
|
3132
|
+
});
|
|
3133
|
+
for (const name of ["pyproject.toml", "uv.lock", "src"]) {
|
|
3134
|
+
await sync(join7(runtime, name), `${root}/runtime/`, true);
|
|
3135
|
+
}
|
|
3136
|
+
for (const file of mappings) {
|
|
3137
|
+
await ssh(`mkdir -p ${shellQuote(file.directory ? file.remote : dirname7(file.remote))}`);
|
|
3138
|
+
if (file.direction !== "output") {
|
|
3139
|
+
const exists3 = await stat5(file.path).catch((error) => {
|
|
3140
|
+
if (error.code === "ENOENT" && file.direction === "both") return void 0;
|
|
3141
|
+
throw error;
|
|
3142
|
+
});
|
|
3143
|
+
if (exists3) await sync(file.path + (file.directory ? "/" : ""), file.remote + (file.directory ? "/" : ""), true);
|
|
3144
|
+
}
|
|
3145
|
+
}
|
|
3146
|
+
if (args.document) {
|
|
3147
|
+
const value = { ...args.document.value };
|
|
3148
|
+
for (const key of args.document.pathKeys) if (typeof value[key] === "string") value[key] = mapPath(value[key]);
|
|
3149
|
+
const path = join7(scratch, "document.json");
|
|
3150
|
+
await writeFile4(path, JSON.stringify(value), { mode: 384 });
|
|
3151
|
+
await sync(path, mapPath(args.document.path), true);
|
|
3152
|
+
}
|
|
3153
|
+
const env = {
|
|
3154
|
+
UV_PROJECT_ENVIRONMENT: `${root}/venv`,
|
|
3155
|
+
TORCH_DISABLE_NATIVE_JIT: "1",
|
|
3156
|
+
HF_HUB_OFFLINE: "1",
|
|
3157
|
+
TRANSFORMERS_OFFLINE: "1"
|
|
3158
|
+
};
|
|
3159
|
+
for (const [key, value] of Object.entries(args.env ?? {})) {
|
|
3160
|
+
if (value === void 0) continue;
|
|
3161
|
+
if (key === "BACKEND") env[key] = value;
|
|
3162
|
+
if (["SM_CHANNEL_TRAINING", "SM_CHANNEL_BASE_MODEL", "SM_MODEL_DIR", "SM_OUTPUT_DIR", "TT_HYPERPARAMETERS_PATH"].includes(key)) env[key] = mapPath(value);
|
|
3163
|
+
}
|
|
3164
|
+
const commandArgs = args.commandArgs.map((value) => value.startsWith("/") ? mapPath(value) : value);
|
|
3165
|
+
const command = ["env", ...Object.entries(env).map(([key, value]) => `${key}=${value}`), args.command, ...commandArgs].map(shellQuote).join(" ");
|
|
3166
|
+
const script = `umask 077; cd ${root}; setsid timeout -k 120 ${args.gpu.maxSeconds}s ${command} & pid=$!; echo "$pid" > pid; trap ${shellQuote(stop)} HUP INT TERM; wait "$pid"`;
|
|
3167
|
+
await args.reporter?.onEvent?.({ stage: args.stage, status: "running", message: `Running GPU process on ${args.gpu.instanceId}.`, details: { remote_directory: root } });
|
|
3168
|
+
started = true;
|
|
3169
|
+
const result = await runLoggedProcess({
|
|
3170
|
+
...args,
|
|
3171
|
+
env: process.env,
|
|
3172
|
+
command: "ssh",
|
|
3173
|
+
commandArgs: [...options, "-tt", host, `bash -lc ${shellQuote(script)}`]
|
|
3174
|
+
});
|
|
3175
|
+
completed = result.exitCode === 0;
|
|
3176
|
+
return result;
|
|
3177
|
+
} finally {
|
|
3178
|
+
try {
|
|
3179
|
+
if (started) {
|
|
3180
|
+
await ssh(`bash -c ${shellQuote(`if test -f ${root}/pid; then pid=$(cat ${root}/pid); ${stop}; fi`)}`, false);
|
|
3181
|
+
for (const file of mappings.filter((file2) => file2.direction !== "input")) {
|
|
3182
|
+
await mkdir6(file.directory ? file.path : dirname7(file.path), { recursive: true });
|
|
3183
|
+
if (!completed && !file.directory) {
|
|
3184
|
+
const exists3 = await runLoggedProcess({ command: "ssh", commandArgs: [...options, host, `test -e ${shellQuote(file.remote)}`], stage: args.stage, timeoutMs: 3e4 });
|
|
3185
|
+
if (exists3.exitCode === 1) continue;
|
|
3186
|
+
if (exists3.exitCode !== 0) throw new Error("Cannot inspect remote outputs.");
|
|
3187
|
+
}
|
|
3188
|
+
await sync(file.remote + (file.directory ? "/" : ""), file.path + (file.directory ? "/" : ""), false, false, file.directory === true && file.direction === "both");
|
|
3189
|
+
}
|
|
3190
|
+
}
|
|
3191
|
+
if (reservationAttempted) await ssh(`if [ "$(cat ${root}/.owner 2>/dev/null)" = ${shellQuote(owner)} ]; then rm -rf -- ${root}; fi`, false);
|
|
3192
|
+
} catch (error) {
|
|
3193
|
+
throw new Error(`GPU cleanup or artifact retrieval failed. Remote files remain at ${host}:${root}. ${error instanceof Error ? error.message : error}`);
|
|
3194
|
+
} finally {
|
|
3195
|
+
await rm2(scratch, { recursive: true, force: true });
|
|
3196
|
+
}
|
|
3197
|
+
}
|
|
3198
|
+
}
|
|
2916
3199
|
|
|
2917
3200
|
// src/local-runtime/evaluation.ts
|
|
2918
|
-
import { createHash as
|
|
2919
|
-
import { mkdir as
|
|
2920
|
-
import { dirname as
|
|
3201
|
+
import { createHash as createHash6 } from "crypto";
|
|
3202
|
+
import { mkdir as mkdir7, readFile as readFile6, rm as rm3, stat as stat6, writeFile as writeFile5 } from "fs/promises";
|
|
3203
|
+
import { dirname as dirname8, join as join8, resolve as resolve11 } from "path";
|
|
2921
3204
|
function normalize(value) {
|
|
2922
3205
|
return value.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2923
3206
|
}
|
|
@@ -3137,8 +3420,8 @@ async function runTransformersInference(args) {
|
|
|
3137
3420
|
const inputPath = `${args.outputPath}.inference-input.json`;
|
|
3138
3421
|
const outputPath = `${args.outputPath}.inference-output.json`;
|
|
3139
3422
|
const logPath = `${args.outputPath}.inference.log`;
|
|
3140
|
-
await
|
|
3141
|
-
await
|
|
3423
|
+
await mkdir7(dirname8(inputPath), { recursive: true });
|
|
3424
|
+
await writeFile5(inputPath, `${JSON.stringify({
|
|
3142
3425
|
protocol_version: INFERENCE_PROTOCOL_VERSION,
|
|
3143
3426
|
kind: args.kind,
|
|
3144
3427
|
model_id: args.modelId,
|
|
@@ -3152,7 +3435,7 @@ async function runTransformersInference(args) {
|
|
|
3152
3435
|
id: String(index),
|
|
3153
3436
|
input: example.input
|
|
3154
3437
|
})),
|
|
3155
|
-
model_cache: args.config.paths.modelCache ?
|
|
3438
|
+
model_cache: args.config.paths.modelCache ? resolve11(args.config.paths.modelCache) : void 0,
|
|
3156
3439
|
trust_remote_code: false,
|
|
3157
3440
|
device: args.config.evaluation.inference.device,
|
|
3158
3441
|
generation: {
|
|
@@ -3162,7 +3445,7 @@ async function runTransformersInference(args) {
|
|
|
3162
3445
|
}
|
|
3163
3446
|
}, null, 2)}
|
|
3164
3447
|
`, "utf8");
|
|
3165
|
-
await
|
|
3448
|
+
await rm3(outputPath, { force: true });
|
|
3166
3449
|
const entrypoint = buildBundledPythonCommand(
|
|
3167
3450
|
"evaluate.py",
|
|
3168
3451
|
["--input", inputPath, "--output", outputPath]
|
|
@@ -3178,7 +3461,30 @@ async function runTransformersInference(args) {
|
|
|
3178
3461
|
log_path: logPath
|
|
3179
3462
|
}
|
|
3180
3463
|
});
|
|
3181
|
-
const
|
|
3464
|
+
const gpu = args.config.evaluation.inference.device === "cuda" ? args.config.gpu : void 0;
|
|
3465
|
+
const execute = gpu ? async (processArgs) => {
|
|
3466
|
+
const document = JSON.parse(await readFile6(inputPath, "utf8"));
|
|
3467
|
+
document.model_source = gpuBaseModelPath(args.config, args.baseModelId, args.baseModelRevision);
|
|
3468
|
+
delete document.model_cache;
|
|
3469
|
+
const files = [
|
|
3470
|
+
{ path: inputPath, direction: "input" },
|
|
3471
|
+
{ path: outputPath, direction: "output" },
|
|
3472
|
+
{ path: document.model_source, direction: "input", directory: true },
|
|
3473
|
+
...document.adapter_path ? [{
|
|
3474
|
+
path: document.adapter_path,
|
|
3475
|
+
direction: "input",
|
|
3476
|
+
directory: (await stat6(document.adapter_path)).isDirectory()
|
|
3477
|
+
}] : []
|
|
3478
|
+
];
|
|
3479
|
+
return runGpuProcess({
|
|
3480
|
+
...processArgs,
|
|
3481
|
+
gpu,
|
|
3482
|
+
runtime: "adapter",
|
|
3483
|
+
files,
|
|
3484
|
+
document: { path: inputPath, value: document, pathKeys: ["model_source", "adapter_path"] }
|
|
3485
|
+
});
|
|
3486
|
+
} : runLoggedProcess;
|
|
3487
|
+
const result = await execute({
|
|
3182
3488
|
command: entrypoint.command,
|
|
3183
3489
|
commandArgs: entrypoint.commandArgs,
|
|
3184
3490
|
env: withBundledPythonEnvironment(
|
|
@@ -3218,7 +3524,7 @@ function assertEvaluationScoringReady(config) {
|
|
|
3218
3524
|
}
|
|
3219
3525
|
}
|
|
3220
3526
|
function baselineCacheKey(args) {
|
|
3221
|
-
return
|
|
3527
|
+
return createHash6("sha256").update(JSON.stringify({
|
|
3222
3528
|
v: 6,
|
|
3223
3529
|
inference_protocol_version: INFERENCE_PROTOCOL_VERSION,
|
|
3224
3530
|
package_version: args.packageVersion,
|
|
@@ -3238,10 +3544,10 @@ function baselineInputsAreStable(args) {
|
|
|
3238
3544
|
return args.config.paths.baseModel ? Boolean(args.sourceFingerprint) : Boolean(args.baseModelRevision);
|
|
3239
3545
|
}
|
|
3240
3546
|
function baselineStoreRoot(config) {
|
|
3241
|
-
return config.storeRoot ?
|
|
3547
|
+
return config.storeRoot ? resolve11(config.storeRoot) : defaultLocalHome();
|
|
3242
3548
|
}
|
|
3243
3549
|
function baselineCachePath(config, key) {
|
|
3244
|
-
return
|
|
3550
|
+
return join8(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
|
|
3245
3551
|
}
|
|
3246
3552
|
async function readBaselineCache(config, path) {
|
|
3247
3553
|
await hardenExistingLocalStore(baselineStoreRoot(config));
|
|
@@ -3255,7 +3561,7 @@ var cachedPackageVersion = null;
|
|
|
3255
3561
|
async function packageVersion() {
|
|
3256
3562
|
if (cachedPackageVersion !== null) return cachedPackageVersion;
|
|
3257
3563
|
try {
|
|
3258
|
-
const raw = await readFile6(
|
|
3564
|
+
const raw = await readFile6(resolve11(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
|
|
3259
3565
|
cachedPackageVersion = String(
|
|
3260
3566
|
JSON.parse(raw).version ?? "unknown"
|
|
3261
3567
|
);
|
|
@@ -3324,7 +3630,7 @@ async function evaluateExamples(args) {
|
|
|
3324
3630
|
baseModelId,
|
|
3325
3631
|
args.baseModelRevision
|
|
3326
3632
|
);
|
|
3327
|
-
const localModelSource = args.modelSource ?
|
|
3633
|
+
const localModelSource = args.modelSource ? resolve11(args.modelSource) !== resolve11(baseModelId) : false;
|
|
3328
3634
|
const baseModelRevision = localModelSource ? defaultBaseModelRevision(baseModelId) : requestedBaseModelRevision;
|
|
3329
3635
|
const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
|
|
3330
3636
|
const truncated = args.examples.length > maxExamples;
|
|
@@ -3440,8 +3746,8 @@ function compareEvalReports(baseline, candidate) {
|
|
|
3440
3746
|
}
|
|
3441
3747
|
|
|
3442
3748
|
// src/local-runtime/process-training.ts
|
|
3443
|
-
import { access, mkdir as
|
|
3444
|
-
import { join as
|
|
3749
|
+
import { access, mkdir as mkdir8, readFile as readFile7, writeFile as writeFile6 } from "fs/promises";
|
|
3750
|
+
import { join as join9, resolve as resolve12 } from "path";
|
|
3445
3751
|
function buildTrainingHyperparameters(request, options = {}) {
|
|
3446
3752
|
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
3447
3753
|
const hyper = request.hyperparameters;
|
|
@@ -3532,15 +3838,15 @@ function createTrainingProgressForwarder(reporter) {
|
|
|
3532
3838
|
async function launchProcessTraining(args) {
|
|
3533
3839
|
const { request, artifacts, config } = args;
|
|
3534
3840
|
const jobName = `tt-local-${request.run_id}`;
|
|
3535
|
-
await
|
|
3841
|
+
await mkdir8(artifacts.trainingDir, { recursive: true });
|
|
3536
3842
|
await copyDatasetToTrainingChannel(artifacts);
|
|
3537
3843
|
const hyperparameters = buildTrainingHyperparameters(request, {
|
|
3538
3844
|
baseModelRevision: args.baseModelRevision
|
|
3539
3845
|
});
|
|
3540
|
-
await writeJson(
|
|
3846
|
+
await writeJson(join9(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
|
|
3541
3847
|
const entrypoint = buildBundledPythonCommand("train.py");
|
|
3542
3848
|
if (config.dryRun) {
|
|
3543
|
-
await
|
|
3849
|
+
await writeFile6(
|
|
3544
3850
|
artifacts.trainingLog,
|
|
3545
3851
|
"Dry run enabled. Training process was not launched.\n",
|
|
3546
3852
|
"utf8"
|
|
@@ -3562,20 +3868,22 @@ async function launchProcessTraining(args) {
|
|
|
3562
3868
|
withOfflineHuggingFaceCacheEnvironment({
|
|
3563
3869
|
...inheritedEnv,
|
|
3564
3870
|
BACKEND: "local",
|
|
3565
|
-
SM_CHANNEL_TRAINING:
|
|
3566
|
-
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ?
|
|
3567
|
-
SM_MODEL_DIR:
|
|
3568
|
-
SM_OUTPUT_DIR:
|
|
3569
|
-
TT_HYPERPARAMETERS_PATH:
|
|
3871
|
+
SM_CHANNEL_TRAINING: resolve12(artifacts.trainingInputDir),
|
|
3872
|
+
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve12(config.paths.baseModel) : void 0,
|
|
3873
|
+
SM_MODEL_DIR: resolve12(artifacts.trainingModelDir),
|
|
3874
|
+
SM_OUTPUT_DIR: resolve12(artifacts.trainingOutputDir),
|
|
3875
|
+
TT_HYPERPARAMETERS_PATH: resolve12(artifacts.trainingConfigDir, "hyperparameters.json")
|
|
3570
3876
|
}, config.paths.modelCache)
|
|
3571
3877
|
);
|
|
3572
3878
|
for (const [key, value] of Object.entries(childEnv)) {
|
|
3573
3879
|
if (value === void 0) delete childEnv[key];
|
|
3574
3880
|
}
|
|
3575
|
-
|
|
3881
|
+
const remoteBaseModel = config.gpu ? gpuBaseModelPath(config, request.spec_snapshot.base_model, args.baseModelRevision) : void 0;
|
|
3882
|
+
if (remoteBaseModel) childEnv.SM_CHANNEL_BASE_MODEL = remoteBaseModel;
|
|
3883
|
+
if (config.paths.modelCache) await mkdir8(resolve12(config.paths.modelCache), { recursive: true });
|
|
3576
3884
|
if (config.paths.baseModel) {
|
|
3577
3885
|
await verifyLocalBaseModel(
|
|
3578
|
-
|
|
3886
|
+
resolve12(config.paths.baseModel),
|
|
3579
3887
|
request.spec_snapshot.base_model,
|
|
3580
3888
|
config.paths.modelCache
|
|
3581
3889
|
);
|
|
@@ -3592,7 +3900,19 @@ async function launchProcessTraining(args) {
|
|
|
3592
3900
|
}
|
|
3593
3901
|
});
|
|
3594
3902
|
const forwardTrainingProgress = createTrainingProgressForwarder(args.reporter);
|
|
3595
|
-
const
|
|
3903
|
+
const execute = config.gpu ? (processArgs) => runGpuProcess({
|
|
3904
|
+
...processArgs,
|
|
3905
|
+
gpu: config.gpu,
|
|
3906
|
+
runtime: "adapter",
|
|
3907
|
+
files: [
|
|
3908
|
+
{ path: artifacts.trainingInputDir, direction: "input", directory: true },
|
|
3909
|
+
{ path: artifacts.trainingConfigDir, direction: "input", directory: true },
|
|
3910
|
+
{ path: remoteBaseModel, direction: "input", directory: true },
|
|
3911
|
+
{ path: artifacts.trainingModelDir, direction: "output", directory: true },
|
|
3912
|
+
{ path: artifacts.trainingOutputDir, direction: "output", directory: true }
|
|
3913
|
+
]
|
|
3914
|
+
}) : runLoggedProcess;
|
|
3915
|
+
const { exitCode } = await execute({
|
|
3596
3916
|
command: entrypoint.command,
|
|
3597
3917
|
commandArgs: entrypoint.commandArgs,
|
|
3598
3918
|
env: childEnv,
|
|
@@ -3602,12 +3922,12 @@ async function launchProcessTraining(args) {
|
|
|
3602
3922
|
onLine: (line) => forwardTrainingProgress(line),
|
|
3603
3923
|
shouldCancel: args.shouldCancel
|
|
3604
3924
|
});
|
|
3605
|
-
const metricsPath =
|
|
3925
|
+
const metricsPath = join9(artifacts.trainingModelDir, "training-metrics.json");
|
|
3606
3926
|
const metrics = await pathExists(metricsPath) ? JSON.parse(await readFile7(metricsPath, "utf8")) : null;
|
|
3607
3927
|
if (exitCode !== 0) {
|
|
3608
3928
|
throw new Error(`Training process exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
|
|
3609
3929
|
}
|
|
3610
|
-
const modelTarPath =
|
|
3930
|
+
const modelTarPath = join9(artifacts.trainingOutputDir, "model.tar.gz");
|
|
3611
3931
|
const modelUri = await pathExists(modelTarPath) ? fileUri(modelTarPath) : fileUri(artifacts.trainingModelDir);
|
|
3612
3932
|
await args.reporter?.onEvent?.({
|
|
3613
3933
|
stage: "training",
|
|
@@ -3621,7 +3941,7 @@ async function launchProcessTraining(args) {
|
|
|
3621
3941
|
}
|
|
3622
3942
|
});
|
|
3623
3943
|
return {
|
|
3624
|
-
provider: "local-uv",
|
|
3944
|
+
provider: config.gpu ? "aws-ssh" : "local-uv",
|
|
3625
3945
|
training_job_name: jobName,
|
|
3626
3946
|
model_artifact_uri: modelUri,
|
|
3627
3947
|
base_model_artifact_uri: config.paths.baseModel ? fileUri(config.paths.baseModel) : void 0,
|
|
@@ -3710,16 +4030,17 @@ async function loadRunRequest(path) {
|
|
|
3710
4030
|
function parseLocalRunnerConfig(document, path) {
|
|
3711
4031
|
const config = localRunnerConfigSchema.parse(document);
|
|
3712
4032
|
if (!path) return config;
|
|
3713
|
-
const configPath =
|
|
3714
|
-
const base =
|
|
4033
|
+
const configPath = resolve13(path);
|
|
4034
|
+
const base = dirname9(configPath);
|
|
3715
4035
|
const configPathValue = (value) => {
|
|
3716
4036
|
if (!value) return void 0;
|
|
3717
4037
|
if (value === "~") return homedir4();
|
|
3718
|
-
if (value.startsWith("~/")) return
|
|
3719
|
-
return
|
|
4038
|
+
if (value.startsWith("~/")) return resolve13(homedir4(), value.slice(2));
|
|
4039
|
+
return resolve13(base, value);
|
|
3720
4040
|
};
|
|
3721
4041
|
return {
|
|
3722
4042
|
...config,
|
|
4043
|
+
gpu: config.gpu ? { ...config.gpu, identityFile: configPathValue(config.gpu.identityFile) } : void 0,
|
|
3723
4044
|
artifactRoot: configPathValue(config.artifactRoot),
|
|
3724
4045
|
storeRoot: configPathValue(config.storeRoot),
|
|
3725
4046
|
paths: {
|
|
@@ -3737,7 +4058,7 @@ function parseLocalRunnerConfig(document, path) {
|
|
|
3737
4058
|
}
|
|
3738
4059
|
async function loadLocalRunnerConfig(path) {
|
|
3739
4060
|
if (!path) return parseLocalRunnerConfig({});
|
|
3740
|
-
const configPath =
|
|
4061
|
+
const configPath = resolve13(path);
|
|
3741
4062
|
return parseLocalRunnerConfig(
|
|
3742
4063
|
await loadJsonFile(configPath),
|
|
3743
4064
|
configPath
|
|
@@ -3752,7 +4073,7 @@ function stripFileUri(path) {
|
|
|
3752
4073
|
}
|
|
3753
4074
|
async function pathExists2(path) {
|
|
3754
4075
|
try {
|
|
3755
|
-
await
|
|
4076
|
+
await stat7(path);
|
|
3756
4077
|
return true;
|
|
3757
4078
|
} catch {
|
|
3758
4079
|
return false;
|
|
@@ -3910,7 +4231,7 @@ function canonicalJson2(value) {
|
|
|
3910
4231
|
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(object[key])}`).join(",")}}`;
|
|
3911
4232
|
}
|
|
3912
4233
|
function hashJson(value) {
|
|
3913
|
-
return
|
|
4234
|
+
return createHash7("sha256").update(canonicalJson2(value)).digest("hex");
|
|
3914
4235
|
}
|
|
3915
4236
|
var packageRoot2 = localRuntimePackageRoot(import.meta.url);
|
|
3916
4237
|
async function hashFileIfPresent(path) {
|
|
@@ -3922,21 +4243,21 @@ async function hashFileIfPresent(path) {
|
|
|
3922
4243
|
}
|
|
3923
4244
|
async function packageVersion2() {
|
|
3924
4245
|
try {
|
|
3925
|
-
const parsed = JSON.parse(await readFile8(
|
|
4246
|
+
const parsed = JSON.parse(await readFile8(resolve13(packageRoot2, "package.json"), "utf8"));
|
|
3926
4247
|
return typeof parsed.version === "string" ? parsed.version : "unknown";
|
|
3927
4248
|
} catch {
|
|
3928
4249
|
return "unknown";
|
|
3929
4250
|
}
|
|
3930
4251
|
}
|
|
3931
4252
|
async function runtimeFingerprint() {
|
|
3932
|
-
const bundledProject2 =
|
|
4253
|
+
const bundledProject2 = resolve13(packageRoot2, "training/adapter");
|
|
3933
4254
|
return hashJson({
|
|
3934
4255
|
tt_local_version: await packageVersion2(),
|
|
3935
4256
|
node_version: process.version,
|
|
3936
4257
|
platform: process.platform,
|
|
3937
4258
|
architecture: process.arch,
|
|
3938
|
-
runner_pyproject: await hashFileIfPresent(
|
|
3939
|
-
runner_uv_lock: await hashFileIfPresent(
|
|
4259
|
+
runner_pyproject: await hashFileIfPresent(resolve13(bundledProject2, "pyproject.toml")),
|
|
4260
|
+
runner_uv_lock: await hashFileIfPresent(resolve13(bundledProject2, "uv.lock"))
|
|
3940
4261
|
});
|
|
3941
4262
|
}
|
|
3942
4263
|
function preparedSourceFingerprint(args) {
|
|
@@ -3950,7 +4271,7 @@ function preparedSourceFingerprint(args) {
|
|
|
3950
4271
|
});
|
|
3951
4272
|
}
|
|
3952
4273
|
async function hashFile(path) {
|
|
3953
|
-
const hash =
|
|
4274
|
+
const hash = createHash7("sha256");
|
|
3954
4275
|
for await (const chunk of createReadStream3(path)) hash.update(chunk);
|
|
3955
4276
|
return hash.digest("hex");
|
|
3956
4277
|
}
|
|
@@ -3983,7 +4304,7 @@ async function resolveBaseModelRevision(request, config) {
|
|
|
3983
4304
|
}
|
|
3984
4305
|
const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
|
|
3985
4306
|
const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
|
|
3986
|
-
const refPath =
|
|
4307
|
+
const refPath = resolve13(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
|
|
3987
4308
|
try {
|
|
3988
4309
|
const revision = (await readFile8(refPath, "utf8")).trim();
|
|
3989
4310
|
if (revision) return revision;
|
|
@@ -4004,12 +4325,12 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
|
|
|
4004
4325
|
const files = [];
|
|
4005
4326
|
const visit = async (directory) => {
|
|
4006
4327
|
for (const entry of await readdir5(directory, { withFileTypes: true })) {
|
|
4007
|
-
const child =
|
|
4328
|
+
const child = resolve13(directory, entry.name);
|
|
4008
4329
|
if (entry.isDirectory()) {
|
|
4009
4330
|
await visit(child);
|
|
4010
4331
|
continue;
|
|
4011
4332
|
}
|
|
4012
|
-
const childMetadata = await
|
|
4333
|
+
const childMetadata = await stat7(child);
|
|
4013
4334
|
if (entry.isSymbolicLink() && !childMetadata.isFile()) {
|
|
4014
4335
|
throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
|
|
4015
4336
|
}
|
|
@@ -4043,27 +4364,27 @@ async function readStageMetadata(path) {
|
|
|
4043
4364
|
}
|
|
4044
4365
|
}
|
|
4045
4366
|
async function removePrefixedArtifacts(path) {
|
|
4046
|
-
const directory =
|
|
4047
|
-
const prefix = `${
|
|
4367
|
+
const directory = dirname9(path);
|
|
4368
|
+
const prefix = `${basename2(path)}.`;
|
|
4048
4369
|
const names = await readdir5(directory).catch(() => []);
|
|
4049
4370
|
await Promise.all([
|
|
4050
|
-
|
|
4051
|
-
...names.filter((name) => name.startsWith(prefix)).map((name) =>
|
|
4371
|
+
rm4(path, { force: true }),
|
|
4372
|
+
...names.filter((name) => name.startsWith(prefix)).map((name) => rm4(resolve13(directory, name), { recursive: true, force: true }))
|
|
4052
4373
|
]);
|
|
4053
4374
|
}
|
|
4054
4375
|
async function cleanupStageArtifacts(artifacts, stage) {
|
|
4055
|
-
const removeReport = async () =>
|
|
4376
|
+
const removeReport = async () => rm4(artifacts.runReportJson, { force: true });
|
|
4056
4377
|
if (stage === "prepare") {
|
|
4057
4378
|
await Promise.all([
|
|
4058
4379
|
removePrefixedArtifacts(artifacts.baselineEvalJson),
|
|
4059
4380
|
removePrefixedArtifacts(artifacts.candidateEvalJson),
|
|
4060
4381
|
removePrefixedArtifacts(artifacts.generalBaselineEvalJson),
|
|
4061
4382
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
4062
|
-
|
|
4383
|
+
rm4(artifacts.trainingDir, { recursive: true, force: true }),
|
|
4063
4384
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
4064
|
-
|
|
4385
|
+
rm4(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
4065
4386
|
removeReport(),
|
|
4066
|
-
|
|
4387
|
+
rm4(artifacts.artifactManifestJson, { force: true })
|
|
4067
4388
|
]);
|
|
4068
4389
|
return;
|
|
4069
4390
|
}
|
|
@@ -4077,9 +4398,9 @@ async function cleanupStageArtifacts(artifacts, stage) {
|
|
|
4077
4398
|
}
|
|
4078
4399
|
if (stage === "train") {
|
|
4079
4400
|
await Promise.all([
|
|
4080
|
-
|
|
4401
|
+
rm4(artifacts.trainingDir, { recursive: true, force: true }),
|
|
4081
4402
|
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
4082
|
-
|
|
4403
|
+
rm4(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
4083
4404
|
removePrefixedArtifacts(artifacts.candidateEvalJson),
|
|
4084
4405
|
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
4085
4406
|
removeReport()
|
|
@@ -4115,8 +4436,8 @@ async function throwIfCancelled(store, request) {
|
|
|
4115
4436
|
}
|
|
4116
4437
|
}
|
|
4117
4438
|
async function acquireWorkflowLock(lockPath, description) {
|
|
4118
|
-
await
|
|
4119
|
-
const token =
|
|
4439
|
+
await mkdir9(dirname9(lockPath), { recursive: true });
|
|
4440
|
+
const token = randomUUID5();
|
|
4120
4441
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4121
4442
|
try {
|
|
4122
4443
|
const handle = await open3(lockPath, "wx", 384);
|
|
@@ -4129,13 +4450,13 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
4129
4450
|
return async () => {
|
|
4130
4451
|
try {
|
|
4131
4452
|
const owner = JSON.parse(await readFile8(lockPath, "utf8"));
|
|
4132
|
-
if (owner.token === token) await
|
|
4453
|
+
if (owner.token === token) await rm4(lockPath, { force: true });
|
|
4133
4454
|
} catch {
|
|
4134
4455
|
}
|
|
4135
4456
|
};
|
|
4136
4457
|
} catch (error) {
|
|
4137
4458
|
if (error.code !== "EEXIST") throw error;
|
|
4138
|
-
const metadata = await
|
|
4459
|
+
const metadata = await stat7(lockPath).catch(() => null);
|
|
4139
4460
|
let owner = null;
|
|
4140
4461
|
try {
|
|
4141
4462
|
owner = JSON.parse(await readFile8(lockPath, "utf8"));
|
|
@@ -4157,10 +4478,10 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
4157
4478
|
`${description} already has an active local workflow (lock: ${lockPath}).`
|
|
4158
4479
|
);
|
|
4159
4480
|
}
|
|
4160
|
-
const stalePath = `${lockPath}.stale.${
|
|
4481
|
+
const stalePath = `${lockPath}.stale.${randomUUID5()}`;
|
|
4161
4482
|
try {
|
|
4162
4483
|
await rename3(lockPath, stalePath);
|
|
4163
|
-
await
|
|
4484
|
+
await rm4(stalePath, { force: true });
|
|
4164
4485
|
} catch (renameError) {
|
|
4165
4486
|
if (renameError.code !== "ENOENT") throw renameError;
|
|
4166
4487
|
}
|
|
@@ -4170,11 +4491,11 @@ async function acquireWorkflowLock(lockPath, description) {
|
|
|
4170
4491
|
}
|
|
4171
4492
|
async function acquireRunLock(store, runId) {
|
|
4172
4493
|
await store.ensure();
|
|
4173
|
-
return acquireWorkflowLock(
|
|
4494
|
+
return acquireWorkflowLock(resolve13(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
|
|
4174
4495
|
}
|
|
4175
4496
|
async function acquireArtifactLock(artifacts) {
|
|
4176
4497
|
return acquireWorkflowLock(
|
|
4177
|
-
|
|
4498
|
+
join10(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
|
|
4178
4499
|
`Artifact directory ${artifacts.runDir}`
|
|
4179
4500
|
);
|
|
4180
4501
|
}
|
|
@@ -4221,13 +4542,13 @@ function stageFingerprintPath(prepared, stage) {
|
|
|
4221
4542
|
return `${stageOutputPath(prepared, stage)}.stage.json`;
|
|
4222
4543
|
}
|
|
4223
4544
|
async function runnerFileFingerprints() {
|
|
4224
|
-
const project =
|
|
4225
|
-
const bundledSource =
|
|
4226
|
-
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) =>
|
|
4545
|
+
const project = resolve13(packageRoot2, "training/adapter");
|
|
4546
|
+
const bundledSource = resolve13(packageRoot2, "training/adapter/src");
|
|
4547
|
+
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve13(bundledSource, entry.name));
|
|
4227
4548
|
const candidates = [.../* @__PURE__ */ new Set([
|
|
4228
4549
|
...pythonSources,
|
|
4229
|
-
|
|
4230
|
-
|
|
4550
|
+
resolve13(project, "pyproject.toml"),
|
|
4551
|
+
resolve13(project, "uv.lock")
|
|
4231
4552
|
])];
|
|
4232
4553
|
const fingerprints = {};
|
|
4233
4554
|
for (const path of candidates.sort()) fingerprints[path] = await hashFileIfPresent(path);
|
|
@@ -4341,7 +4662,7 @@ async function ensureRunRecord(args) {
|
|
|
4341
4662
|
`Run ${args.request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
|
|
4342
4663
|
);
|
|
4343
4664
|
}
|
|
4344
|
-
if (
|
|
4665
|
+
if (resolve13(existing.artifact_dir) !== resolve13(args.artifacts.runDir)) {
|
|
4345
4666
|
throw new Error(
|
|
4346
4667
|
`Run ${args.request.run_id} is already bound to artifact directory ${existing.artifact_dir}; it cannot be resumed with ${args.artifacts.runDir}.`
|
|
4347
4668
|
);
|
|
@@ -4419,7 +4740,7 @@ async function computePreparedRun(args) {
|
|
|
4419
4740
|
base_model_for_evaluation: baseModelForEvaluation,
|
|
4420
4741
|
base_model_revision: baseModelRevision ?? null,
|
|
4421
4742
|
base_model_fingerprint: baseModelFingerprint ?? null,
|
|
4422
|
-
system_prompt_sha256:
|
|
4743
|
+
system_prompt_sha256: createHash7("sha256").update(system).digest("hex"),
|
|
4423
4744
|
prepared_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
4424
4745
|
};
|
|
4425
4746
|
if (args.writeArtifacts) await writeJsonAtomic(artifacts.stageMetadataJson, metadata);
|
|
@@ -4448,7 +4769,7 @@ async function prepareStage(args) {
|
|
|
4448
4769
|
const preparedExists = await pathExists2(args.artifacts.stageMetadataJson) && await pathExists2(args.artifacts.trainingJsonl);
|
|
4449
4770
|
const prepared = await computePreparedRun({ ...args, writeArtifacts: false });
|
|
4450
4771
|
await args.store.syncRunRequest(args.request, args.artifacts.runDir);
|
|
4451
|
-
await writeJsonAtomic(
|
|
4772
|
+
await writeJsonAtomic(resolve13(args.artifacts.runDir, "request.json"), args.request);
|
|
4452
4773
|
const existingMetadata = preparedExists ? await readStageMetadata(args.artifacts.stageMetadataJson) : null;
|
|
4453
4774
|
let canReuse = preparedExists && existingMetadata?.source_fingerprint === prepared.metadata.source_fingerprint;
|
|
4454
4775
|
if (canReuse) {
|
|
@@ -4617,8 +4938,8 @@ async function runTrainStage(args) {
|
|
|
4617
4938
|
request: args.prepared.request,
|
|
4618
4939
|
status: "training",
|
|
4619
4940
|
stage: "training",
|
|
4620
|
-
message: args.config.dryRun ? "Recording dry-run training result." : "Launching local training process.",
|
|
4621
|
-
details: { training_backend: "local-uv", dry_run: args.config.dryRun }
|
|
4941
|
+
message: args.config.dryRun ? "Recording dry-run training result." : args.config.gpu ? "Launching training on your AWS GPU." : "Launching local training process.",
|
|
4942
|
+
details: { training_backend: args.config.gpu && !args.config.dryRun ? "aws-ssh" : "local-uv", dry_run: args.config.dryRun }
|
|
4622
4943
|
});
|
|
4623
4944
|
const training = await launchProcessTraining({
|
|
4624
4945
|
request: args.prepared.request,
|
|
@@ -4985,7 +5306,7 @@ async function runLocalPipeline(input) {
|
|
|
4985
5306
|
if (step.uses === "evaluate") {
|
|
4986
5307
|
const isBase = step.with.model === "base";
|
|
4987
5308
|
const report = isBase ? await runBaselineStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter }) : await runCandidateStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
|
|
4988
|
-
const reportPath =
|
|
5309
|
+
const reportPath = join10(artifacts.runDir, "evaluations", `${step.id}.json`);
|
|
4989
5310
|
await writeJsonAtomic(reportPath, { ...report, artifact_uri: fileUri(reportPath) });
|
|
4990
5311
|
outputs[step.id] = { report: { ...report, artifact_uri: fileUri(reportPath) }, reportPath };
|
|
4991
5312
|
await refreshArtifactManifest(prepared);
|
|
@@ -5306,10 +5627,7 @@ function evaluateCapabilities(inventory) {
|
|
|
5306
5627
|
train,
|
|
5307
5628
|
finetune: train,
|
|
5308
5629
|
inference: hasCuda ? inference : { ...inference, reason: `CPU eval: ${inference.reason}` },
|
|
5309
|
-
serve: {
|
|
5310
|
-
status: "not_possible",
|
|
5311
|
-
reason: "tt serve cannot host foundation checkpoints yet"
|
|
5312
|
-
}
|
|
5630
|
+
serve: hasCuda && inventory.os.platform === "linux" ? inference : { status: "not_possible", reason: "Foundation serving requires Linux and an NVIDIA CUDA GPU" }
|
|
5313
5631
|
};
|
|
5314
5632
|
return { cuda_available: hasCuda, gpu, adapters, foundation, notes };
|
|
5315
5633
|
}
|
|
@@ -5323,7 +5641,7 @@ function formatCapabilitySummary(report) {
|
|
|
5323
5641
|
return [
|
|
5324
5642
|
`GPU ${gpuLabel}, CUDA ${report.cuda_available ? "yes" : "no"}`,
|
|
5325
5643
|
`Adapter: ${adapterBits.join("; ")}`,
|
|
5326
|
-
`Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve
|
|
5644
|
+
`Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve ${report.foundation.serve.status}`
|
|
5327
5645
|
].join(". ");
|
|
5328
5646
|
}
|
|
5329
5647
|
function warningsFromSnapshot(report, target) {
|
|
@@ -5360,7 +5678,7 @@ function warningsFromSnapshot(report, target) {
|
|
|
5360
5678
|
import { constants as constants2 } from "fs";
|
|
5361
5679
|
import { access as access2, statfs } from "fs/promises";
|
|
5362
5680
|
import { cpus, freemem, homedir as homedir5, totalmem, type } from "os";
|
|
5363
|
-
import { dirname as
|
|
5681
|
+
import { dirname as dirname10, join as join11, resolve as resolve14 } from "path";
|
|
5364
5682
|
import { spawn as spawn2 } from "child_process";
|
|
5365
5683
|
async function runCommand(command, args, options = {}) {
|
|
5366
5684
|
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
@@ -5514,7 +5832,7 @@ async function queryNvidiaGpus(env) {
|
|
|
5514
5832
|
return { result: table.code === 0 || !last ? table : last, gpus };
|
|
5515
5833
|
}
|
|
5516
5834
|
async function diskCheck(name, path) {
|
|
5517
|
-
const resolvedPath =
|
|
5835
|
+
const resolvedPath = resolve14(path);
|
|
5518
5836
|
try {
|
|
5519
5837
|
await access2(resolvedPath, constants2.R_OK);
|
|
5520
5838
|
const fs = await statfs(resolvedPath);
|
|
@@ -5529,7 +5847,7 @@ async function diskCheck(name, path) {
|
|
|
5529
5847
|
} catch (error) {
|
|
5530
5848
|
let probe = resolvedPath;
|
|
5531
5849
|
for (let i = 0; i < 6; i += 1) {
|
|
5532
|
-
const parent =
|
|
5850
|
+
const parent = dirname10(probe);
|
|
5533
5851
|
if (parent === probe) break;
|
|
5534
5852
|
probe = parent;
|
|
5535
5853
|
try {
|
|
@@ -5615,10 +5933,10 @@ async function collectHostInventory(options = {}) {
|
|
|
5615
5933
|
const cwd = options.cwd ?? process.cwd();
|
|
5616
5934
|
const nodeVersion = process.versions.node;
|
|
5617
5935
|
const nodeMajor = Number(nodeVersion.split(".")[0]);
|
|
5618
|
-
const artifactRoot =
|
|
5619
|
-
const storeRoot =
|
|
5620
|
-
const modelCache =
|
|
5621
|
-
config?.paths.modelCache ?? env.HF_HOME ??
|
|
5936
|
+
const artifactRoot = resolve14(cwd, config?.artifactRoot ?? join11(cwd, DEFAULT_ARTIFACT_ROOT));
|
|
5937
|
+
const storeRoot = resolve14(config?.storeRoot ?? defaultStoreRoot(env));
|
|
5938
|
+
const modelCache = resolve14(
|
|
5939
|
+
config?.paths.modelCache ?? env.HF_HOME ?? join11(homedir5(), ".cache", "huggingface")
|
|
5622
5940
|
);
|
|
5623
5941
|
const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
|
|
5624
5942
|
diskCheck("artifact-root", artifactRoot),
|
|
@@ -5664,12 +5982,12 @@ async function collectHostInventory(options = {}) {
|
|
|
5664
5982
|
}
|
|
5665
5983
|
|
|
5666
5984
|
// src/local-runtime/hardware-snapshot.ts
|
|
5667
|
-
import { chmod as chmod2, mkdir as
|
|
5668
|
-
import { dirname as
|
|
5985
|
+
import { chmod as chmod2, mkdir as mkdir10, readFile as readFile9, writeFile as writeFile7 } from "fs/promises";
|
|
5986
|
+
import { dirname as dirname11, join as join12 } from "path";
|
|
5669
5987
|
var HARDWARE_SNAPSHOT_VERSION = 1;
|
|
5670
5988
|
var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
5671
5989
|
function hardwareSnapshotPath(env = process.env) {
|
|
5672
|
-
return
|
|
5990
|
+
return join12(getTunedTensorHome(env), "hardware.json");
|
|
5673
5991
|
}
|
|
5674
5992
|
async function readHardwareSnapshot(env = process.env) {
|
|
5675
5993
|
const path = hardwareSnapshotPath(env);
|
|
@@ -5691,7 +6009,7 @@ async function readHardwareSnapshot(env = process.env) {
|
|
|
5691
6009
|
}
|
|
5692
6010
|
async function writeHardwareSnapshot(snapshot, env = process.env) {
|
|
5693
6011
|
const path = hardwareSnapshotPath(env);
|
|
5694
|
-
await
|
|
6012
|
+
await mkdir10(dirname11(path), { recursive: true, mode: 448 });
|
|
5695
6013
|
const gpu = snapshot.capabilities.gpu;
|
|
5696
6014
|
const record = {
|
|
5697
6015
|
version: HARDWARE_SNAPSHOT_VERSION,
|
|
@@ -5702,7 +6020,7 @@ async function writeHardwareSnapshot(snapshot, env = process.env) {
|
|
|
5702
6020
|
capabilities: snapshot.capabilities,
|
|
5703
6021
|
summary: snapshot.summary ?? formatCapabilitySummary(snapshot.capabilities)
|
|
5704
6022
|
};
|
|
5705
|
-
await
|
|
6023
|
+
await writeFile7(path, `${JSON.stringify(record, null, 2)}
|
|
5706
6024
|
`, { mode: 384 });
|
|
5707
6025
|
await chmod2(path, 384);
|
|
5708
6026
|
return record;
|
|
@@ -5809,6 +6127,8 @@ export {
|
|
|
5809
6127
|
runRequestFromLocalSpec,
|
|
5810
6128
|
parseLocalRunInput,
|
|
5811
6129
|
loadLocalRunInput,
|
|
6130
|
+
specHash,
|
|
6131
|
+
validateBehaviorSpec,
|
|
5812
6132
|
normalizeChatJsonlForRelocation,
|
|
5813
6133
|
buildSystemMessage,
|
|
5814
6134
|
exampleToChatRow,
|
|
@@ -5825,15 +6145,17 @@ export {
|
|
|
5825
6145
|
buildFoundationPythonCommand,
|
|
5826
6146
|
withFoundationPythonEnvironment,
|
|
5827
6147
|
runLoggedProcess,
|
|
6148
|
+
minimalMachineLearningEnvironment,
|
|
6149
|
+
withHuggingFaceCacheEnvironment,
|
|
6150
|
+
withOfflineHuggingFaceCacheEnvironment,
|
|
6151
|
+
checkAwsGpu,
|
|
6152
|
+
runGpuProcess,
|
|
5828
6153
|
isTerminalRunState,
|
|
5829
6154
|
defaultLocalHome,
|
|
5830
6155
|
localStorePaths,
|
|
5831
6156
|
hardenExistingLocalStore,
|
|
5832
6157
|
writePrivateJsonAtomic,
|
|
5833
6158
|
createLocalStore,
|
|
5834
|
-
minimalMachineLearningEnvironment,
|
|
5835
|
-
withHuggingFaceCacheEnvironment,
|
|
5836
|
-
withOfflineHuggingFaceCacheEnvironment,
|
|
5837
6159
|
tokenF1,
|
|
5838
6160
|
verifyModelPrefetchCacheReport,
|
|
5839
6161
|
buildModelPrefetchPayload,
|
|
@@ -5856,4 +6178,4 @@ export {
|
|
|
5856
6178
|
formatAgentHostBlock,
|
|
5857
6179
|
assessHardware
|
|
5858
6180
|
};
|
|
5859
|
-
//# sourceMappingURL=chunk-
|
|
6181
|
+
//# sourceMappingURL=chunk-KRKISPHG.js.map
|