@tuned-tensor/cli 0.15.1 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1076,7 +1076,7 @@ var comparisonReportSchema = z.object({
1076
1076
  }).strict())
1077
1077
  }).strict();
1078
1078
  var trainingReportSchema = z.object({
1079
- provider: z.literal("local-uv"),
1079
+ provider: z.enum(["local-uv", "aws-ssh"]),
1080
1080
  training_job_name: z.string(),
1081
1081
  model_artifact_uri: z.string().optional(),
1082
1082
  base_model_artifact_uri: z.string().optional(),
@@ -1167,6 +1167,16 @@ var evaluationConfigSchema = z.object({
1167
1167
  }).strict().optional()
1168
1168
  }).strict();
1169
1169
  var localRunnerConfigSchema = z.object({
1170
+ gpu: z.object({
1171
+ provider: z.literal("aws"),
1172
+ instanceId: z.string().regex(/^i-[a-f0-9]{8,17}$/),
1173
+ profile: z.string().min(1).optional(),
1174
+ region: z.string().min(1).optional(),
1175
+ user: z.string().regex(/^[a-z_][a-z0-9_-]*$/i),
1176
+ identityFile: z.string().min(1).optional(),
1177
+ privateIp: z.boolean().default(false),
1178
+ maxSeconds: z.number().int().min(60).max(604800).default(86400)
1179
+ }).strict().optional(),
1170
1180
  storeRoot: z.string().optional(),
1171
1181
  artifactRoot: z.string().default(DEFAULT_ARTIFACT_ROOT),
1172
1182
  dryRun: z.boolean().default(false),
@@ -1428,6 +1438,56 @@ async function loadLocalRunInput(path, options = {}) {
1428
1438
  options
1429
1439
  );
1430
1440
  }
1441
+ function specHash(source) {
1442
+ return createHash2("sha256").update(source).digest("hex");
1443
+ }
1444
+ function validateBehaviorSpec(input) {
1445
+ const errors = [];
1446
+ const warnings = [];
1447
+ if (input.kind === "request") {
1448
+ return { valid: false, errors: ["Use a behavior spec, not a run request."], warnings };
1449
+ }
1450
+ const { spec } = input;
1451
+ const placeholders = input.kind === "foundation-spec" ? foundationPlaceholderIssues(input.spec) : generatedPlaceholderIssues(input.request);
1452
+ if (placeholders.length) {
1453
+ errors.push(`Edit the generated behavior spec before training: ${placeholders.join("; ")}.`);
1454
+ }
1455
+ if (input.kind === "foundation-spec") {
1456
+ if (input.spec.foundation.nproc_per_node !== 1) {
1457
+ errors.push("Foundation nproc_per_node must be 1 for this runtime.");
1458
+ }
1459
+ const nonNumericOutput = spec.examples.some(
1460
+ (example) => !/-?\d+(?:\.\d+)?/.test(example.output.replaceAll(",", ""))
1461
+ );
1462
+ if (input.spec.foundation.rl_steps > 0 && nonNumericOutput) {
1463
+ errors.push("Foundation RL requires numeric expected outputs in every example.");
1464
+ }
1465
+ }
1466
+ for (const key of ["name", "system_prompt"]) {
1467
+ if (!spec[key].trim()) errors.push(`${key} must not be blank.`);
1468
+ }
1469
+ if (spec.examples.length < 2 && !(input.kind === "spec" && input.spec.dataset_prebuilt)) {
1470
+ errors.push("Add at least two examples for training and evaluation.");
1471
+ }
1472
+ const seen = /* @__PURE__ */ new Set();
1473
+ for (const [index, example] of spec.examples.entries()) {
1474
+ if (!example.input.trim() || !example.output.trim()) {
1475
+ errors.push(`examples[${index}] input and output must not be blank.`);
1476
+ }
1477
+ const key = example.input.trim();
1478
+ if (seen.has(key)) {
1479
+ errors.push(`examples[${index}] repeats an input; use distinct examples to avoid conflicting labels or train/eval overlap.`);
1480
+ }
1481
+ seen.add(key);
1482
+ }
1483
+ if (!spec.guidelines.length) {
1484
+ warnings.push("Add guidelines to make expected behavior explicit.");
1485
+ }
1486
+ if (spec.examples.length < 10) {
1487
+ warnings.push("This is a small example set; passing validation does not establish model quality.");
1488
+ }
1489
+ return { valid: errors.length === 0, errors, warnings };
1490
+ }
1431
1491
 
1432
1492
  // src/local-runtime/dataset.ts
1433
1493
  import { readFile as readFile3 } from "fs/promises";
@@ -2907,17 +2967,240 @@ function evaluateGeneralRegressionGate(comparison, policy) {
2907
2967
  }
2908
2968
 
2909
2969
  // src/local-runtime/orchestrator.ts
2910
- import { lstat as lstat5, mkdir as mkdir8, open as open3, readFile as readFile8, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
2970
+ import { lstat as lstat5, mkdir as mkdir9, open as open3, readFile as readFile8, readdir as readdir5, rename as rename3, rm as rm4, stat as stat7 } from "fs/promises";
2911
2971
  import { createReadStream as createReadStream3 } from "fs";
2912
- import { createHash as createHash6, randomUUID as randomUUID4 } from "crypto";
2972
+ import { createHash as createHash7, randomUUID as randomUUID5 } from "crypto";
2913
2973
  import { performance } from "perf_hooks";
2914
2974
  import { homedir as homedir4 } from "os";
2915
- import { basename, dirname as dirname8, join as join9, relative as relative4, resolve as resolve12 } from "path";
2975
+ import { basename as basename2, dirname as dirname9, join as join10, relative as relative4, resolve as resolve13 } from "path";
2976
+
2977
+ // src/local-runtime/gpu-executor.ts
2978
+ import { createHash as createHash5, randomUUID as randomUUID4 } from "crypto";
2979
+ import { mkdtemp, mkdir as mkdir6, realpath as realpath3, rm as rm2, stat as stat5, writeFile as writeFile4 } from "fs/promises";
2980
+ import { tmpdir } from "os";
2981
+ import { basename, dirname as dirname7, join as join7, resolve as resolve10 } from "path";
2982
+ function shellQuote(value) {
2983
+ return `'${value.replaceAll("'", "'\\''")}'`;
2984
+ }
2985
+ async function resolveGpuHost(gpu) {
2986
+ let stdout = "";
2987
+ const result = await runLoggedProcess({
2988
+ command: "aws",
2989
+ commandArgs: [
2990
+ "ec2",
2991
+ "describe-instances",
2992
+ "--instance-ids",
2993
+ gpu.instanceId,
2994
+ ...gpu.profile ? ["--profile", gpu.profile] : [],
2995
+ ...gpu.region ? ["--region", gpu.region] : [],
2996
+ "--output",
2997
+ "json",
2998
+ "--no-cli-pager"
2999
+ ],
3000
+ timeoutMs: 3e4,
3001
+ stage: "gpu_connect",
3002
+ onLine: (line, stream) => {
3003
+ if (stream === "stdout") stdout += line;
3004
+ }
3005
+ });
3006
+ if (result.exitCode !== 0) throw new Error(`AWS GPU lookup failed: ${result.stderr}`);
3007
+ const instances = JSON.parse(stdout).Reservations?.flatMap((r) => r.Instances ?? []);
3008
+ const instance = instances?.find((i) => i.InstanceId === gpu.instanceId);
3009
+ if (instance?.State?.Name !== "running") throw new Error(`AWS GPU instance ${gpu.instanceId} must already be running.`);
3010
+ const host = gpu.privateIp ? instance.PrivateIpAddress : instance.PublicIpAddress;
3011
+ if (typeof host !== "string" || !/^[0-9.]+$/.test(host)) {
3012
+ throw new Error("AWS GPU instance has no selected IP address. Use privateIp with a reachable VPC/VPN, or assign a public IP.");
3013
+ }
3014
+ return `${gpu.user}@${host}`;
3015
+ }
3016
+ function sshOptions(gpu) {
3017
+ return [
3018
+ "-o",
3019
+ "BatchMode=yes",
3020
+ "-o",
3021
+ "StrictHostKeyChecking=yes",
3022
+ "-o",
3023
+ "ConnectTimeout=15",
3024
+ "-o",
3025
+ "ServerAliveInterval=15",
3026
+ "-o",
3027
+ "ServerAliveCountMax=3",
3028
+ ...gpu.identityFile ? ["-i", gpu.identityFile] : []
3029
+ ];
3030
+ }
3031
+ async function checkAwsGpu(gpu) {
3032
+ const host = await resolveGpuHost(gpu);
3033
+ const result = await runLoggedProcess({
3034
+ command: "ssh",
3035
+ commandArgs: [
3036
+ ...sshOptions(gpu),
3037
+ host,
3038
+ `bash -lc ${shellQuote("command -v uv && command -v rsync && command -v timeout && command -v setsid && nvidia-smi")}`
3039
+ ],
3040
+ stage: "gpu_check",
3041
+ timeoutMs: 3e4
3042
+ });
3043
+ if (result.exitCode !== 0) throw new Error(`AWS GPU preflight failed: ${result.stderr}`);
3044
+ }
3045
+ function gpuBaseModelPath(config, model, revision) {
3046
+ if (config.paths.baseModel) return resolve10(config.paths.baseModel);
3047
+ if (!revision || !/^[a-f0-9]{40}$/i.test(revision)) {
3048
+ throw new Error("AWS GPU execution needs an immutable base-model revision. Run tt models prefetch first or configure paths.baseModel.");
3049
+ }
3050
+ const env = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
3051
+ return join7(env.HF_HUB_CACHE, `models--${model.replaceAll("/", "--")}`, "snapshots", revision);
3052
+ }
3053
+ async function physicalTransferPath(path) {
3054
+ try {
3055
+ return await realpath3(path);
3056
+ } catch (error) {
3057
+ if (error.code !== "ENOENT" || dirname7(path) === path) throw error;
3058
+ return join7(await physicalTransferPath(dirname7(path)), basename(path));
3059
+ }
3060
+ }
3061
+ async function assertSeparateTransferPaths(files) {
3062
+ const paths = await Promise.all(files.map((file) => physicalTransferPath(resolve10(file.path))));
3063
+ for (let i = 0; i < files.length; i += 1) {
3064
+ for (let j = 0; j < i; j += 1) {
3065
+ if (files[i].direction === "input" && files[j].direction === "input") continue;
3066
+ const left = paths[i];
3067
+ const right = paths[j];
3068
+ if (left === right || files[i].directory && right.startsWith(`${left}/`) || files[j].directory && left.startsWith(`${right}/`)) {
3069
+ throw new Error(`GPU transfer paths overlap: ${files[i].path} and ${files[j].path}. Use separate input, output, recovery, and backup directories.`);
3070
+ }
3071
+ }
3072
+ }
3073
+ }
3074
+ async function runGpuProcess(args) {
3075
+ if (await args.shouldCancel?.()) throw new ProcessCancelledError();
3076
+ await assertSeparateTransferPaths(args.files);
3077
+ const host = await resolveGpuHost(args.gpu);
3078
+ const options = sshOptions(args.gpu);
3079
+ const identity = args.logPath ? createHash5("sha256").update(resolve10(args.logPath)).digest("hex").slice(0, 32) : randomUUID4();
3080
+ const root = `/tmp/tt-gpu-${identity}`;
3081
+ const scratch = await mkdtemp(join7(tmpdir(), "tt-gpu-"));
3082
+ const runtime = join7(localRuntimePackageRoot(import.meta.url), "training", args.runtime);
3083
+ const mappings = args.files.map((file, index) => ({ ...file, path: resolve10(file.path), remote: `${root}/files/${index}/${basename(file.path)}` }));
3084
+ const mapPath = (path) => {
3085
+ const absolute = resolve10(path);
3086
+ const match = [...mappings].sort((a, b) => b.path.length - a.path.length).find((file) => absolute === file.path || file.directory && absolute.startsWith(`${file.path}/`));
3087
+ if (match) return match.remote + absolute.slice(match.path.length);
3088
+ if (absolute === runtime || absolute.startsWith(`${runtime}/`)) return `${root}/runtime${absolute.slice(runtime.length)}`;
3089
+ throw new Error(`GPU process path is not declared for transfer: ${path}`);
3090
+ };
3091
+ const transport = async (command, commandArgs, cancellable = true) => {
3092
+ const result = await runLoggedProcess({
3093
+ command,
3094
+ commandArgs,
3095
+ stage: args.stage,
3096
+ reporter: args.reporter,
3097
+ timeoutMs: 30 * 6e4,
3098
+ shouldCancel: cancellable ? args.shouldCancel : void 0
3099
+ });
3100
+ if (result.exitCode !== 0) throw new Error(`${command} failed for ${host}: ${result.stderr}`);
3101
+ };
3102
+ const ssh = async (script, cancellable = true) => transport("ssh", [...options, host, script], cancellable);
3103
+ const sync = async (source, destination, upload, cancellable = true, mirror = false) => {
3104
+ await transport("rsync", [
3105
+ "-rlt",
3106
+ "--copy-links",
3107
+ "--protect-args",
3108
+ "--modify-window=-1",
3109
+ "--chmod=Du=rwx,Dgo=,Fu=rw,Fgo=",
3110
+ ...!upload ? ["--delay-updates", ...mirror ? ["--delete-delay"] : []] : [],
3111
+ "--exclude=.venv",
3112
+ "--exclude=__pycache__",
3113
+ "--exclude=.git",
3114
+ "-e",
3115
+ ["ssh", ...options].map(shellQuote).join(" "),
3116
+ "--",
3117
+ upload ? source : `${host}:${source}`,
3118
+ upload ? `${host}:${destination}` : destination
3119
+ ], cancellable);
3120
+ };
3121
+ const grace = Math.ceil((args.shutdownGraceMs ?? 5e3) / 1e3);
3122
+ const stop = `kill -TERM -- -"$pid" 2>/dev/null || true; remaining=${grace}; while kill -0 -- -"$pid" 2>/dev/null && [ "$remaining" -gt 0 ]; do sleep 1; remaining=$((remaining - 1)); done; kill -KILL -- -"$pid" 2>/dev/null || true`;
3123
+ const owner = randomUUID4();
3124
+ let reservationAttempted = false;
3125
+ let started = false;
3126
+ let completed = false;
3127
+ try {
3128
+ reservationAttempted = true;
3129
+ await ssh(`umask 077; mkdir ${root} && printf '%s' ${shellQuote(owner)} > ${root}/.owner && mkdir ${root}/runtime ${root}/files`).catch((error) => {
3130
+ if (error instanceof ProcessCancelledError) throw error;
3131
+ throw new Error(`Cannot reserve GPU staging directory ${host}:${root}. Another process or unrecovered artifacts may be present. ${error instanceof Error ? error.message : error}`);
3132
+ });
3133
+ for (const name of ["pyproject.toml", "uv.lock", "src"]) {
3134
+ await sync(join7(runtime, name), `${root}/runtime/`, true);
3135
+ }
3136
+ for (const file of mappings) {
3137
+ await ssh(`mkdir -p ${shellQuote(file.directory ? file.remote : dirname7(file.remote))}`);
3138
+ if (file.direction !== "output") {
3139
+ const exists3 = await stat5(file.path).catch((error) => {
3140
+ if (error.code === "ENOENT" && file.direction === "both") return void 0;
3141
+ throw error;
3142
+ });
3143
+ if (exists3) await sync(file.path + (file.directory ? "/" : ""), file.remote + (file.directory ? "/" : ""), true);
3144
+ }
3145
+ }
3146
+ if (args.document) {
3147
+ const value = { ...args.document.value };
3148
+ for (const key of args.document.pathKeys) if (typeof value[key] === "string") value[key] = mapPath(value[key]);
3149
+ const path = join7(scratch, "document.json");
3150
+ await writeFile4(path, JSON.stringify(value), { mode: 384 });
3151
+ await sync(path, mapPath(args.document.path), true);
3152
+ }
3153
+ const env = {
3154
+ UV_PROJECT_ENVIRONMENT: `${root}/venv`,
3155
+ TORCH_DISABLE_NATIVE_JIT: "1",
3156
+ HF_HUB_OFFLINE: "1",
3157
+ TRANSFORMERS_OFFLINE: "1"
3158
+ };
3159
+ for (const [key, value] of Object.entries(args.env ?? {})) {
3160
+ if (value === void 0) continue;
3161
+ if (key === "BACKEND") env[key] = value;
3162
+ if (["SM_CHANNEL_TRAINING", "SM_CHANNEL_BASE_MODEL", "SM_MODEL_DIR", "SM_OUTPUT_DIR", "TT_HYPERPARAMETERS_PATH"].includes(key)) env[key] = mapPath(value);
3163
+ }
3164
+ const commandArgs = args.commandArgs.map((value) => value.startsWith("/") ? mapPath(value) : value);
3165
+ const command = ["env", ...Object.entries(env).map(([key, value]) => `${key}=${value}`), args.command, ...commandArgs].map(shellQuote).join(" ");
3166
+ const script = `umask 077; cd ${root}; setsid timeout -k 120 ${args.gpu.maxSeconds}s ${command} & pid=$!; echo "$pid" > pid; trap ${shellQuote(stop)} HUP INT TERM; wait "$pid"`;
3167
+ await args.reporter?.onEvent?.({ stage: args.stage, status: "running", message: `Running GPU process on ${args.gpu.instanceId}.`, details: { remote_directory: root } });
3168
+ started = true;
3169
+ const result = await runLoggedProcess({
3170
+ ...args,
3171
+ env: process.env,
3172
+ command: "ssh",
3173
+ commandArgs: [...options, "-tt", host, `bash -lc ${shellQuote(script)}`]
3174
+ });
3175
+ completed = result.exitCode === 0;
3176
+ return result;
3177
+ } finally {
3178
+ try {
3179
+ if (started) {
3180
+ await ssh(`bash -c ${shellQuote(`if test -f ${root}/pid; then pid=$(cat ${root}/pid); ${stop}; fi`)}`, false);
3181
+ for (const file of mappings.filter((file2) => file2.direction !== "input")) {
3182
+ await mkdir6(file.directory ? file.path : dirname7(file.path), { recursive: true });
3183
+ if (!completed && !file.directory) {
3184
+ const exists3 = await runLoggedProcess({ command: "ssh", commandArgs: [...options, host, `test -e ${shellQuote(file.remote)}`], stage: args.stage, timeoutMs: 3e4 });
3185
+ if (exists3.exitCode === 1) continue;
3186
+ if (exists3.exitCode !== 0) throw new Error("Cannot inspect remote outputs.");
3187
+ }
3188
+ await sync(file.remote + (file.directory ? "/" : ""), file.path + (file.directory ? "/" : ""), false, false, file.directory === true && file.direction === "both");
3189
+ }
3190
+ }
3191
+ if (reservationAttempted) await ssh(`if [ "$(cat ${root}/.owner 2>/dev/null)" = ${shellQuote(owner)} ]; then rm -rf -- ${root}; fi`, false);
3192
+ } catch (error) {
3193
+ throw new Error(`GPU cleanup or artifact retrieval failed. Remote files remain at ${host}:${root}. ${error instanceof Error ? error.message : error}`);
3194
+ } finally {
3195
+ await rm2(scratch, { recursive: true, force: true });
3196
+ }
3197
+ }
3198
+ }
2916
3199
 
2917
3200
  // src/local-runtime/evaluation.ts
2918
- import { createHash as createHash5 } from "crypto";
2919
- import { mkdir as mkdir6, readFile as readFile6, rm as rm2, writeFile as writeFile4 } from "fs/promises";
2920
- import { dirname as dirname7, join as join7, resolve as resolve10 } from "path";
3201
+ import { createHash as createHash6 } from "crypto";
3202
+ import { mkdir as mkdir7, readFile as readFile6, rm as rm3, stat as stat6, writeFile as writeFile5 } from "fs/promises";
3203
+ import { dirname as dirname8, join as join8, resolve as resolve11 } from "path";
2921
3204
  function normalize(value) {
2922
3205
  return value.trim().replace(/\s+/g, " ").toLowerCase();
2923
3206
  }
@@ -3137,8 +3420,8 @@ async function runTransformersInference(args) {
3137
3420
  const inputPath = `${args.outputPath}.inference-input.json`;
3138
3421
  const outputPath = `${args.outputPath}.inference-output.json`;
3139
3422
  const logPath = `${args.outputPath}.inference.log`;
3140
- await mkdir6(dirname7(inputPath), { recursive: true });
3141
- await writeFile4(inputPath, `${JSON.stringify({
3423
+ await mkdir7(dirname8(inputPath), { recursive: true });
3424
+ await writeFile5(inputPath, `${JSON.stringify({
3142
3425
  protocol_version: INFERENCE_PROTOCOL_VERSION,
3143
3426
  kind: args.kind,
3144
3427
  model_id: args.modelId,
@@ -3152,7 +3435,7 @@ async function runTransformersInference(args) {
3152
3435
  id: String(index),
3153
3436
  input: example.input
3154
3437
  })),
3155
- model_cache: args.config.paths.modelCache ? resolve10(args.config.paths.modelCache) : void 0,
3438
+ model_cache: args.config.paths.modelCache ? resolve11(args.config.paths.modelCache) : void 0,
3156
3439
  trust_remote_code: false,
3157
3440
  device: args.config.evaluation.inference.device,
3158
3441
  generation: {
@@ -3162,7 +3445,7 @@ async function runTransformersInference(args) {
3162
3445
  }
3163
3446
  }, null, 2)}
3164
3447
  `, "utf8");
3165
- await rm2(outputPath, { force: true });
3448
+ await rm3(outputPath, { force: true });
3166
3449
  const entrypoint = buildBundledPythonCommand(
3167
3450
  "evaluate.py",
3168
3451
  ["--input", inputPath, "--output", outputPath]
@@ -3178,7 +3461,30 @@ async function runTransformersInference(args) {
3178
3461
  log_path: logPath
3179
3462
  }
3180
3463
  });
3181
- const result = await runLoggedProcess({
3464
+ const gpu = args.config.evaluation.inference.device === "cuda" ? args.config.gpu : void 0;
3465
+ const execute = gpu ? async (processArgs) => {
3466
+ const document = JSON.parse(await readFile6(inputPath, "utf8"));
3467
+ document.model_source = gpuBaseModelPath(args.config, args.baseModelId, args.baseModelRevision);
3468
+ delete document.model_cache;
3469
+ const files = [
3470
+ { path: inputPath, direction: "input" },
3471
+ { path: outputPath, direction: "output" },
3472
+ { path: document.model_source, direction: "input", directory: true },
3473
+ ...document.adapter_path ? [{
3474
+ path: document.adapter_path,
3475
+ direction: "input",
3476
+ directory: (await stat6(document.adapter_path)).isDirectory()
3477
+ }] : []
3478
+ ];
3479
+ return runGpuProcess({
3480
+ ...processArgs,
3481
+ gpu,
3482
+ runtime: "adapter",
3483
+ files,
3484
+ document: { path: inputPath, value: document, pathKeys: ["model_source", "adapter_path"] }
3485
+ });
3486
+ } : runLoggedProcess;
3487
+ const result = await execute({
3182
3488
  command: entrypoint.command,
3183
3489
  commandArgs: entrypoint.commandArgs,
3184
3490
  env: withBundledPythonEnvironment(
@@ -3218,7 +3524,7 @@ function assertEvaluationScoringReady(config) {
3218
3524
  }
3219
3525
  }
3220
3526
  function baselineCacheKey(args) {
3221
- return createHash5("sha256").update(JSON.stringify({
3527
+ return createHash6("sha256").update(JSON.stringify({
3222
3528
  v: 6,
3223
3529
  inference_protocol_version: INFERENCE_PROTOCOL_VERSION,
3224
3530
  package_version: args.packageVersion,
@@ -3238,10 +3544,10 @@ function baselineInputsAreStable(args) {
3238
3544
  return args.config.paths.baseModel ? Boolean(args.sourceFingerprint) : Boolean(args.baseModelRevision);
3239
3545
  }
3240
3546
  function baselineStoreRoot(config) {
3241
- return config.storeRoot ? resolve10(config.storeRoot) : defaultLocalHome();
3547
+ return config.storeRoot ? resolve11(config.storeRoot) : defaultLocalHome();
3242
3548
  }
3243
3549
  function baselineCachePath(config, key) {
3244
- return join7(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
3550
+ return join8(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
3245
3551
  }
3246
3552
  async function readBaselineCache(config, path) {
3247
3553
  await hardenExistingLocalStore(baselineStoreRoot(config));
@@ -3255,7 +3561,7 @@ var cachedPackageVersion = null;
3255
3561
  async function packageVersion() {
3256
3562
  if (cachedPackageVersion !== null) return cachedPackageVersion;
3257
3563
  try {
3258
- const raw = await readFile6(resolve10(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
3564
+ const raw = await readFile6(resolve11(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
3259
3565
  cachedPackageVersion = String(
3260
3566
  JSON.parse(raw).version ?? "unknown"
3261
3567
  );
@@ -3324,7 +3630,7 @@ async function evaluateExamples(args) {
3324
3630
  baseModelId,
3325
3631
  args.baseModelRevision
3326
3632
  );
3327
- const localModelSource = args.modelSource ? resolve10(args.modelSource) !== resolve10(baseModelId) : false;
3633
+ const localModelSource = args.modelSource ? resolve11(args.modelSource) !== resolve11(baseModelId) : false;
3328
3634
  const baseModelRevision = localModelSource ? defaultBaseModelRevision(baseModelId) : requestedBaseModelRevision;
3329
3635
  const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
3330
3636
  const truncated = args.examples.length > maxExamples;
@@ -3440,8 +3746,8 @@ function compareEvalReports(baseline, candidate) {
3440
3746
  }
3441
3747
 
3442
3748
  // src/local-runtime/process-training.ts
3443
- import { access, mkdir as mkdir7, readFile as readFile7, writeFile as writeFile5 } from "fs/promises";
3444
- import { join as join8, resolve as resolve11 } from "path";
3749
+ import { access, mkdir as mkdir8, readFile as readFile7, writeFile as writeFile6 } from "fs/promises";
3750
+ import { join as join9, resolve as resolve12 } from "path";
3445
3751
  function buildTrainingHyperparameters(request, options = {}) {
3446
3752
  const model = resolveTrainingModel(request.spec_snapshot.base_model);
3447
3753
  const hyper = request.hyperparameters;
@@ -3532,15 +3838,15 @@ function createTrainingProgressForwarder(reporter) {
3532
3838
  async function launchProcessTraining(args) {
3533
3839
  const { request, artifacts, config } = args;
3534
3840
  const jobName = `tt-local-${request.run_id}`;
3535
- await mkdir7(artifacts.trainingDir, { recursive: true });
3841
+ await mkdir8(artifacts.trainingDir, { recursive: true });
3536
3842
  await copyDatasetToTrainingChannel(artifacts);
3537
3843
  const hyperparameters = buildTrainingHyperparameters(request, {
3538
3844
  baseModelRevision: args.baseModelRevision
3539
3845
  });
3540
- await writeJson(join8(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
3846
+ await writeJson(join9(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
3541
3847
  const entrypoint = buildBundledPythonCommand("train.py");
3542
3848
  if (config.dryRun) {
3543
- await writeFile5(
3849
+ await writeFile6(
3544
3850
  artifacts.trainingLog,
3545
3851
  "Dry run enabled. Training process was not launched.\n",
3546
3852
  "utf8"
@@ -3562,20 +3868,22 @@ async function launchProcessTraining(args) {
3562
3868
  withOfflineHuggingFaceCacheEnvironment({
3563
3869
  ...inheritedEnv,
3564
3870
  BACKEND: "local",
3565
- SM_CHANNEL_TRAINING: resolve11(artifacts.trainingInputDir),
3566
- SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve11(config.paths.baseModel) : void 0,
3567
- SM_MODEL_DIR: resolve11(artifacts.trainingModelDir),
3568
- SM_OUTPUT_DIR: resolve11(artifacts.trainingOutputDir),
3569
- TT_HYPERPARAMETERS_PATH: resolve11(artifacts.trainingConfigDir, "hyperparameters.json")
3871
+ SM_CHANNEL_TRAINING: resolve12(artifacts.trainingInputDir),
3872
+ SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve12(config.paths.baseModel) : void 0,
3873
+ SM_MODEL_DIR: resolve12(artifacts.trainingModelDir),
3874
+ SM_OUTPUT_DIR: resolve12(artifacts.trainingOutputDir),
3875
+ TT_HYPERPARAMETERS_PATH: resolve12(artifacts.trainingConfigDir, "hyperparameters.json")
3570
3876
  }, config.paths.modelCache)
3571
3877
  );
3572
3878
  for (const [key, value] of Object.entries(childEnv)) {
3573
3879
  if (value === void 0) delete childEnv[key];
3574
3880
  }
3575
- if (config.paths.modelCache) await mkdir7(resolve11(config.paths.modelCache), { recursive: true });
3881
+ const remoteBaseModel = config.gpu ? gpuBaseModelPath(config, request.spec_snapshot.base_model, args.baseModelRevision) : void 0;
3882
+ if (remoteBaseModel) childEnv.SM_CHANNEL_BASE_MODEL = remoteBaseModel;
3883
+ if (config.paths.modelCache) await mkdir8(resolve12(config.paths.modelCache), { recursive: true });
3576
3884
  if (config.paths.baseModel) {
3577
3885
  await verifyLocalBaseModel(
3578
- resolve11(config.paths.baseModel),
3886
+ resolve12(config.paths.baseModel),
3579
3887
  request.spec_snapshot.base_model,
3580
3888
  config.paths.modelCache
3581
3889
  );
@@ -3592,7 +3900,19 @@ async function launchProcessTraining(args) {
3592
3900
  }
3593
3901
  });
3594
3902
  const forwardTrainingProgress = createTrainingProgressForwarder(args.reporter);
3595
- const { exitCode } = await runLoggedProcess({
3903
+ const execute = config.gpu ? (processArgs) => runGpuProcess({
3904
+ ...processArgs,
3905
+ gpu: config.gpu,
3906
+ runtime: "adapter",
3907
+ files: [
3908
+ { path: artifacts.trainingInputDir, direction: "input", directory: true },
3909
+ { path: artifacts.trainingConfigDir, direction: "input", directory: true },
3910
+ { path: remoteBaseModel, direction: "input", directory: true },
3911
+ { path: artifacts.trainingModelDir, direction: "output", directory: true },
3912
+ { path: artifacts.trainingOutputDir, direction: "output", directory: true }
3913
+ ]
3914
+ }) : runLoggedProcess;
3915
+ const { exitCode } = await execute({
3596
3916
  command: entrypoint.command,
3597
3917
  commandArgs: entrypoint.commandArgs,
3598
3918
  env: childEnv,
@@ -3602,12 +3922,12 @@ async function launchProcessTraining(args) {
3602
3922
  onLine: (line) => forwardTrainingProgress(line),
3603
3923
  shouldCancel: args.shouldCancel
3604
3924
  });
3605
- const metricsPath = join8(artifacts.trainingModelDir, "training-metrics.json");
3925
+ const metricsPath = join9(artifacts.trainingModelDir, "training-metrics.json");
3606
3926
  const metrics = await pathExists(metricsPath) ? JSON.parse(await readFile7(metricsPath, "utf8")) : null;
3607
3927
  if (exitCode !== 0) {
3608
3928
  throw new Error(`Training process exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
3609
3929
  }
3610
- const modelTarPath = join8(artifacts.trainingOutputDir, "model.tar.gz");
3930
+ const modelTarPath = join9(artifacts.trainingOutputDir, "model.tar.gz");
3611
3931
  const modelUri = await pathExists(modelTarPath) ? fileUri(modelTarPath) : fileUri(artifacts.trainingModelDir);
3612
3932
  await args.reporter?.onEvent?.({
3613
3933
  stage: "training",
@@ -3621,7 +3941,7 @@ async function launchProcessTraining(args) {
3621
3941
  }
3622
3942
  });
3623
3943
  return {
3624
- provider: "local-uv",
3944
+ provider: config.gpu ? "aws-ssh" : "local-uv",
3625
3945
  training_job_name: jobName,
3626
3946
  model_artifact_uri: modelUri,
3627
3947
  base_model_artifact_uri: config.paths.baseModel ? fileUri(config.paths.baseModel) : void 0,
@@ -3710,16 +4030,17 @@ async function loadRunRequest(path) {
3710
4030
  function parseLocalRunnerConfig(document, path) {
3711
4031
  const config = localRunnerConfigSchema.parse(document);
3712
4032
  if (!path) return config;
3713
- const configPath = resolve12(path);
3714
- const base = dirname8(configPath);
4033
+ const configPath = resolve13(path);
4034
+ const base = dirname9(configPath);
3715
4035
  const configPathValue = (value) => {
3716
4036
  if (!value) return void 0;
3717
4037
  if (value === "~") return homedir4();
3718
- if (value.startsWith("~/")) return resolve12(homedir4(), value.slice(2));
3719
- return resolve12(base, value);
4038
+ if (value.startsWith("~/")) return resolve13(homedir4(), value.slice(2));
4039
+ return resolve13(base, value);
3720
4040
  };
3721
4041
  return {
3722
4042
  ...config,
4043
+ gpu: config.gpu ? { ...config.gpu, identityFile: configPathValue(config.gpu.identityFile) } : void 0,
3723
4044
  artifactRoot: configPathValue(config.artifactRoot),
3724
4045
  storeRoot: configPathValue(config.storeRoot),
3725
4046
  paths: {
@@ -3737,7 +4058,7 @@ function parseLocalRunnerConfig(document, path) {
3737
4058
  }
3738
4059
  async function loadLocalRunnerConfig(path) {
3739
4060
  if (!path) return parseLocalRunnerConfig({});
3740
- const configPath = resolve12(path);
4061
+ const configPath = resolve13(path);
3741
4062
  return parseLocalRunnerConfig(
3742
4063
  await loadJsonFile(configPath),
3743
4064
  configPath
@@ -3752,7 +4073,7 @@ function stripFileUri(path) {
3752
4073
  }
3753
4074
  async function pathExists2(path) {
3754
4075
  try {
3755
- await stat5(path);
4076
+ await stat7(path);
3756
4077
  return true;
3757
4078
  } catch {
3758
4079
  return false;
@@ -3910,7 +4231,7 @@ function canonicalJson2(value) {
3910
4231
  return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(object[key])}`).join(",")}}`;
3911
4232
  }
3912
4233
  function hashJson(value) {
3913
- return createHash6("sha256").update(canonicalJson2(value)).digest("hex");
4234
+ return createHash7("sha256").update(canonicalJson2(value)).digest("hex");
3914
4235
  }
3915
4236
  var packageRoot2 = localRuntimePackageRoot(import.meta.url);
3916
4237
  async function hashFileIfPresent(path) {
@@ -3922,21 +4243,21 @@ async function hashFileIfPresent(path) {
3922
4243
  }
3923
4244
  async function packageVersion2() {
3924
4245
  try {
3925
- const parsed = JSON.parse(await readFile8(resolve12(packageRoot2, "package.json"), "utf8"));
4246
+ const parsed = JSON.parse(await readFile8(resolve13(packageRoot2, "package.json"), "utf8"));
3926
4247
  return typeof parsed.version === "string" ? parsed.version : "unknown";
3927
4248
  } catch {
3928
4249
  return "unknown";
3929
4250
  }
3930
4251
  }
3931
4252
  async function runtimeFingerprint() {
3932
- const bundledProject2 = resolve12(packageRoot2, "training/adapter");
4253
+ const bundledProject2 = resolve13(packageRoot2, "training/adapter");
3933
4254
  return hashJson({
3934
4255
  tt_local_version: await packageVersion2(),
3935
4256
  node_version: process.version,
3936
4257
  platform: process.platform,
3937
4258
  architecture: process.arch,
3938
- runner_pyproject: await hashFileIfPresent(resolve12(bundledProject2, "pyproject.toml")),
3939
- runner_uv_lock: await hashFileIfPresent(resolve12(bundledProject2, "uv.lock"))
4259
+ runner_pyproject: await hashFileIfPresent(resolve13(bundledProject2, "pyproject.toml")),
4260
+ runner_uv_lock: await hashFileIfPresent(resolve13(bundledProject2, "uv.lock"))
3940
4261
  });
3941
4262
  }
3942
4263
  function preparedSourceFingerprint(args) {
@@ -3950,7 +4271,7 @@ function preparedSourceFingerprint(args) {
3950
4271
  });
3951
4272
  }
3952
4273
  async function hashFile(path) {
3953
- const hash = createHash6("sha256");
4274
+ const hash = createHash7("sha256");
3954
4275
  for await (const chunk of createReadStream3(path)) hash.update(chunk);
3955
4276
  return hash.digest("hex");
3956
4277
  }
@@ -3983,7 +4304,7 @@ async function resolveBaseModelRevision(request, config) {
3983
4304
  }
3984
4305
  const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
3985
4306
  const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
3986
- const refPath = resolve12(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
4307
+ const refPath = resolve13(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
3987
4308
  try {
3988
4309
  const revision = (await readFile8(refPath, "utf8")).trim();
3989
4310
  if (revision) return revision;
@@ -4004,12 +4325,12 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
4004
4325
  const files = [];
4005
4326
  const visit = async (directory) => {
4006
4327
  for (const entry of await readdir5(directory, { withFileTypes: true })) {
4007
- const child = resolve12(directory, entry.name);
4328
+ const child = resolve13(directory, entry.name);
4008
4329
  if (entry.isDirectory()) {
4009
4330
  await visit(child);
4010
4331
  continue;
4011
4332
  }
4012
- const childMetadata = await stat5(child);
4333
+ const childMetadata = await stat7(child);
4013
4334
  if (entry.isSymbolicLink() && !childMetadata.isFile()) {
4014
4335
  throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
4015
4336
  }
@@ -4043,27 +4364,27 @@ async function readStageMetadata(path) {
4043
4364
  }
4044
4365
  }
4045
4366
  async function removePrefixedArtifacts(path) {
4046
- const directory = dirname8(path);
4047
- const prefix = `${basename(path)}.`;
4367
+ const directory = dirname9(path);
4368
+ const prefix = `${basename2(path)}.`;
4048
4369
  const names = await readdir5(directory).catch(() => []);
4049
4370
  await Promise.all([
4050
- rm3(path, { force: true }),
4051
- ...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(resolve12(directory, name), { recursive: true, force: true }))
4371
+ rm4(path, { force: true }),
4372
+ ...names.filter((name) => name.startsWith(prefix)).map((name) => rm4(resolve13(directory, name), { recursive: true, force: true }))
4052
4373
  ]);
4053
4374
  }
4054
4375
  async function cleanupStageArtifacts(artifacts, stage) {
4055
- const removeReport = async () => rm3(artifacts.runReportJson, { force: true });
4376
+ const removeReport = async () => rm4(artifacts.runReportJson, { force: true });
4056
4377
  if (stage === "prepare") {
4057
4378
  await Promise.all([
4058
4379
  removePrefixedArtifacts(artifacts.baselineEvalJson),
4059
4380
  removePrefixedArtifacts(artifacts.candidateEvalJson),
4060
4381
  removePrefixedArtifacts(artifacts.generalBaselineEvalJson),
4061
4382
  removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
4062
- rm3(artifacts.trainingDir, { recursive: true, force: true }),
4383
+ rm4(artifacts.trainingDir, { recursive: true, force: true }),
4063
4384
  removePrefixedArtifacts(artifacts.trainingReportJson),
4064
- rm3(resolve12(artifacts.runDir, "model.tar.gz"), { force: true }),
4385
+ rm4(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
4065
4386
  removeReport(),
4066
- rm3(artifacts.artifactManifestJson, { force: true })
4387
+ rm4(artifacts.artifactManifestJson, { force: true })
4067
4388
  ]);
4068
4389
  return;
4069
4390
  }
@@ -4077,9 +4398,9 @@ async function cleanupStageArtifacts(artifacts, stage) {
4077
4398
  }
4078
4399
  if (stage === "train") {
4079
4400
  await Promise.all([
4080
- rm3(artifacts.trainingDir, { recursive: true, force: true }),
4401
+ rm4(artifacts.trainingDir, { recursive: true, force: true }),
4081
4402
  removePrefixedArtifacts(artifacts.trainingReportJson),
4082
- rm3(resolve12(artifacts.runDir, "model.tar.gz"), { force: true }),
4403
+ rm4(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
4083
4404
  removePrefixedArtifacts(artifacts.candidateEvalJson),
4084
4405
  removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
4085
4406
  removeReport()
@@ -4115,8 +4436,8 @@ async function throwIfCancelled(store, request) {
4115
4436
  }
4116
4437
  }
4117
4438
  async function acquireWorkflowLock(lockPath, description) {
4118
- await mkdir8(dirname8(lockPath), { recursive: true });
4119
- const token = randomUUID4();
4439
+ await mkdir9(dirname9(lockPath), { recursive: true });
4440
+ const token = randomUUID5();
4120
4441
  for (let attempt = 0; attempt < 3; attempt += 1) {
4121
4442
  try {
4122
4443
  const handle = await open3(lockPath, "wx", 384);
@@ -4129,13 +4450,13 @@ async function acquireWorkflowLock(lockPath, description) {
4129
4450
  return async () => {
4130
4451
  try {
4131
4452
  const owner = JSON.parse(await readFile8(lockPath, "utf8"));
4132
- if (owner.token === token) await rm3(lockPath, { force: true });
4453
+ if (owner.token === token) await rm4(lockPath, { force: true });
4133
4454
  } catch {
4134
4455
  }
4135
4456
  };
4136
4457
  } catch (error) {
4137
4458
  if (error.code !== "EEXIST") throw error;
4138
- const metadata = await stat5(lockPath).catch(() => null);
4459
+ const metadata = await stat7(lockPath).catch(() => null);
4139
4460
  let owner = null;
4140
4461
  try {
4141
4462
  owner = JSON.parse(await readFile8(lockPath, "utf8"));
@@ -4157,10 +4478,10 @@ async function acquireWorkflowLock(lockPath, description) {
4157
4478
  `${description} already has an active local workflow (lock: ${lockPath}).`
4158
4479
  );
4159
4480
  }
4160
- const stalePath = `${lockPath}.stale.${randomUUID4()}`;
4481
+ const stalePath = `${lockPath}.stale.${randomUUID5()}`;
4161
4482
  try {
4162
4483
  await rename3(lockPath, stalePath);
4163
- await rm3(stalePath, { force: true });
4484
+ await rm4(stalePath, { force: true });
4164
4485
  } catch (renameError) {
4165
4486
  if (renameError.code !== "ENOENT") throw renameError;
4166
4487
  }
@@ -4170,11 +4491,11 @@ async function acquireWorkflowLock(lockPath, description) {
4170
4491
  }
4171
4492
  async function acquireRunLock(store, runId) {
4172
4493
  await store.ensure();
4173
- return acquireWorkflowLock(resolve12(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
4494
+ return acquireWorkflowLock(resolve13(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
4174
4495
  }
4175
4496
  async function acquireArtifactLock(artifacts) {
4176
4497
  return acquireWorkflowLock(
4177
- join9(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
4498
+ join10(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
4178
4499
  `Artifact directory ${artifacts.runDir}`
4179
4500
  );
4180
4501
  }
@@ -4221,13 +4542,13 @@ function stageFingerprintPath(prepared, stage) {
4221
4542
  return `${stageOutputPath(prepared, stage)}.stage.json`;
4222
4543
  }
4223
4544
  async function runnerFileFingerprints() {
4224
- const project = resolve12(packageRoot2, "training/adapter");
4225
- const bundledSource = resolve12(packageRoot2, "training/adapter/src");
4226
- const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve12(bundledSource, entry.name));
4545
+ const project = resolve13(packageRoot2, "training/adapter");
4546
+ const bundledSource = resolve13(packageRoot2, "training/adapter/src");
4547
+ const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve13(bundledSource, entry.name));
4227
4548
  const candidates = [.../* @__PURE__ */ new Set([
4228
4549
  ...pythonSources,
4229
- resolve12(project, "pyproject.toml"),
4230
- resolve12(project, "uv.lock")
4550
+ resolve13(project, "pyproject.toml"),
4551
+ resolve13(project, "uv.lock")
4231
4552
  ])];
4232
4553
  const fingerprints = {};
4233
4554
  for (const path of candidates.sort()) fingerprints[path] = await hashFileIfPresent(path);
@@ -4341,7 +4662,7 @@ async function ensureRunRecord(args) {
4341
4662
  `Run ${args.request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
4342
4663
  );
4343
4664
  }
4344
- if (resolve12(existing.artifact_dir) !== resolve12(args.artifacts.runDir)) {
4665
+ if (resolve13(existing.artifact_dir) !== resolve13(args.artifacts.runDir)) {
4345
4666
  throw new Error(
4346
4667
  `Run ${args.request.run_id} is already bound to artifact directory ${existing.artifact_dir}; it cannot be resumed with ${args.artifacts.runDir}.`
4347
4668
  );
@@ -4419,7 +4740,7 @@ async function computePreparedRun(args) {
4419
4740
  base_model_for_evaluation: baseModelForEvaluation,
4420
4741
  base_model_revision: baseModelRevision ?? null,
4421
4742
  base_model_fingerprint: baseModelFingerprint ?? null,
4422
- system_prompt_sha256: createHash6("sha256").update(system).digest("hex"),
4743
+ system_prompt_sha256: createHash7("sha256").update(system).digest("hex"),
4423
4744
  prepared_at: (/* @__PURE__ */ new Date()).toISOString()
4424
4745
  };
4425
4746
  if (args.writeArtifacts) await writeJsonAtomic(artifacts.stageMetadataJson, metadata);
@@ -4448,7 +4769,7 @@ async function prepareStage(args) {
4448
4769
  const preparedExists = await pathExists2(args.artifacts.stageMetadataJson) && await pathExists2(args.artifacts.trainingJsonl);
4449
4770
  const prepared = await computePreparedRun({ ...args, writeArtifacts: false });
4450
4771
  await args.store.syncRunRequest(args.request, args.artifacts.runDir);
4451
- await writeJsonAtomic(resolve12(args.artifacts.runDir, "request.json"), args.request);
4772
+ await writeJsonAtomic(resolve13(args.artifacts.runDir, "request.json"), args.request);
4452
4773
  const existingMetadata = preparedExists ? await readStageMetadata(args.artifacts.stageMetadataJson) : null;
4453
4774
  let canReuse = preparedExists && existingMetadata?.source_fingerprint === prepared.metadata.source_fingerprint;
4454
4775
  if (canReuse) {
@@ -4617,8 +4938,8 @@ async function runTrainStage(args) {
4617
4938
  request: args.prepared.request,
4618
4939
  status: "training",
4619
4940
  stage: "training",
4620
- message: args.config.dryRun ? "Recording dry-run training result." : "Launching local training process.",
4621
- details: { training_backend: "local-uv", dry_run: args.config.dryRun }
4941
+ message: args.config.dryRun ? "Recording dry-run training result." : args.config.gpu ? "Launching training on your AWS GPU." : "Launching local training process.",
4942
+ details: { training_backend: args.config.gpu && !args.config.dryRun ? "aws-ssh" : "local-uv", dry_run: args.config.dryRun }
4622
4943
  });
4623
4944
  const training = await launchProcessTraining({
4624
4945
  request: args.prepared.request,
@@ -4985,7 +5306,7 @@ async function runLocalPipeline(input) {
4985
5306
  if (step.uses === "evaluate") {
4986
5307
  const isBase = step.with.model === "base";
4987
5308
  const report = isBase ? await runBaselineStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter }) : await runCandidateStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
4988
- const reportPath = join9(artifacts.runDir, "evaluations", `${step.id}.json`);
5309
+ const reportPath = join10(artifacts.runDir, "evaluations", `${step.id}.json`);
4989
5310
  await writeJsonAtomic(reportPath, { ...report, artifact_uri: fileUri(reportPath) });
4990
5311
  outputs[step.id] = { report: { ...report, artifact_uri: fileUri(reportPath) }, reportPath };
4991
5312
  await refreshArtifactManifest(prepared);
@@ -5306,10 +5627,7 @@ function evaluateCapabilities(inventory) {
5306
5627
  train,
5307
5628
  finetune: train,
5308
5629
  inference: hasCuda ? inference : { ...inference, reason: `CPU eval: ${inference.reason}` },
5309
- serve: {
5310
- status: "not_possible",
5311
- reason: "tt serve cannot host foundation checkpoints yet"
5312
- }
5630
+ serve: hasCuda && inventory.os.platform === "linux" ? inference : { status: "not_possible", reason: "Foundation serving requires Linux and an NVIDIA CUDA GPU" }
5313
5631
  };
5314
5632
  return { cuda_available: hasCuda, gpu, adapters, foundation, notes };
5315
5633
  }
@@ -5323,7 +5641,7 @@ function formatCapabilitySummary(report) {
5323
5641
  return [
5324
5642
  `GPU ${gpuLabel}, CUDA ${report.cuda_available ? "yes" : "no"}`,
5325
5643
  `Adapter: ${adapterBits.join("; ")}`,
5326
- `Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve not supported`
5644
+ `Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve ${report.foundation.serve.status}`
5327
5645
  ].join(". ");
5328
5646
  }
5329
5647
  function warningsFromSnapshot(report, target) {
@@ -5360,7 +5678,7 @@ function warningsFromSnapshot(report, target) {
5360
5678
  import { constants as constants2 } from "fs";
5361
5679
  import { access as access2, statfs } from "fs/promises";
5362
5680
  import { cpus, freemem, homedir as homedir5, totalmem, type } from "os";
5363
- import { dirname as dirname9, join as join10, resolve as resolve13 } from "path";
5681
+ import { dirname as dirname10, join as join11, resolve as resolve14 } from "path";
5364
5682
  import { spawn as spawn2 } from "child_process";
5365
5683
  async function runCommand(command, args, options = {}) {
5366
5684
  const timeoutMs = options.timeoutMs ?? 1e4;
@@ -5514,7 +5832,7 @@ async function queryNvidiaGpus(env) {
5514
5832
  return { result: table.code === 0 || !last ? table : last, gpus };
5515
5833
  }
5516
5834
  async function diskCheck(name, path) {
5517
- const resolvedPath = resolve13(path);
5835
+ const resolvedPath = resolve14(path);
5518
5836
  try {
5519
5837
  await access2(resolvedPath, constants2.R_OK);
5520
5838
  const fs = await statfs(resolvedPath);
@@ -5529,7 +5847,7 @@ async function diskCheck(name, path) {
5529
5847
  } catch (error) {
5530
5848
  let probe = resolvedPath;
5531
5849
  for (let i = 0; i < 6; i += 1) {
5532
- const parent = dirname9(probe);
5850
+ const parent = dirname10(probe);
5533
5851
  if (parent === probe) break;
5534
5852
  probe = parent;
5535
5853
  try {
@@ -5615,10 +5933,10 @@ async function collectHostInventory(options = {}) {
5615
5933
  const cwd = options.cwd ?? process.cwd();
5616
5934
  const nodeVersion = process.versions.node;
5617
5935
  const nodeMajor = Number(nodeVersion.split(".")[0]);
5618
- const artifactRoot = resolve13(cwd, config?.artifactRoot ?? join10(cwd, DEFAULT_ARTIFACT_ROOT));
5619
- const storeRoot = resolve13(config?.storeRoot ?? defaultStoreRoot(env));
5620
- const modelCache = resolve13(
5621
- config?.paths.modelCache ?? env.HF_HOME ?? join10(homedir5(), ".cache", "huggingface")
5936
+ const artifactRoot = resolve14(cwd, config?.artifactRoot ?? join11(cwd, DEFAULT_ARTIFACT_ROOT));
5937
+ const storeRoot = resolve14(config?.storeRoot ?? defaultStoreRoot(env));
5938
+ const modelCache = resolve14(
5939
+ config?.paths.modelCache ?? env.HF_HOME ?? join11(homedir5(), ".cache", "huggingface")
5622
5940
  );
5623
5941
  const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
5624
5942
  diskCheck("artifact-root", artifactRoot),
@@ -5664,12 +5982,12 @@ async function collectHostInventory(options = {}) {
5664
5982
  }
5665
5983
 
5666
5984
  // src/local-runtime/hardware-snapshot.ts
5667
- import { chmod as chmod2, mkdir as mkdir9, readFile as readFile9, writeFile as writeFile6 } from "fs/promises";
5668
- import { dirname as dirname10, join as join11 } from "path";
5985
+ import { chmod as chmod2, mkdir as mkdir10, readFile as readFile9, writeFile as writeFile7 } from "fs/promises";
5986
+ import { dirname as dirname11, join as join12 } from "path";
5669
5987
  var HARDWARE_SNAPSHOT_VERSION = 1;
5670
5988
  var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
5671
5989
  function hardwareSnapshotPath(env = process.env) {
5672
- return join11(getTunedTensorHome(env), "hardware.json");
5990
+ return join12(getTunedTensorHome(env), "hardware.json");
5673
5991
  }
5674
5992
  async function readHardwareSnapshot(env = process.env) {
5675
5993
  const path = hardwareSnapshotPath(env);
@@ -5691,7 +6009,7 @@ async function readHardwareSnapshot(env = process.env) {
5691
6009
  }
5692
6010
  async function writeHardwareSnapshot(snapshot, env = process.env) {
5693
6011
  const path = hardwareSnapshotPath(env);
5694
- await mkdir9(dirname10(path), { recursive: true, mode: 448 });
6012
+ await mkdir10(dirname11(path), { recursive: true, mode: 448 });
5695
6013
  const gpu = snapshot.capabilities.gpu;
5696
6014
  const record = {
5697
6015
  version: HARDWARE_SNAPSHOT_VERSION,
@@ -5702,7 +6020,7 @@ async function writeHardwareSnapshot(snapshot, env = process.env) {
5702
6020
  capabilities: snapshot.capabilities,
5703
6021
  summary: snapshot.summary ?? formatCapabilitySummary(snapshot.capabilities)
5704
6022
  };
5705
- await writeFile6(path, `${JSON.stringify(record, null, 2)}
6023
+ await writeFile7(path, `${JSON.stringify(record, null, 2)}
5706
6024
  `, { mode: 384 });
5707
6025
  await chmod2(path, 384);
5708
6026
  return record;
@@ -5809,6 +6127,8 @@ export {
5809
6127
  runRequestFromLocalSpec,
5810
6128
  parseLocalRunInput,
5811
6129
  loadLocalRunInput,
6130
+ specHash,
6131
+ validateBehaviorSpec,
5812
6132
  normalizeChatJsonlForRelocation,
5813
6133
  buildSystemMessage,
5814
6134
  exampleToChatRow,
@@ -5825,15 +6145,17 @@ export {
5825
6145
  buildFoundationPythonCommand,
5826
6146
  withFoundationPythonEnvironment,
5827
6147
  runLoggedProcess,
6148
+ minimalMachineLearningEnvironment,
6149
+ withHuggingFaceCacheEnvironment,
6150
+ withOfflineHuggingFaceCacheEnvironment,
6151
+ checkAwsGpu,
6152
+ runGpuProcess,
5828
6153
  isTerminalRunState,
5829
6154
  defaultLocalHome,
5830
6155
  localStorePaths,
5831
6156
  hardenExistingLocalStore,
5832
6157
  writePrivateJsonAtomic,
5833
6158
  createLocalStore,
5834
- minimalMachineLearningEnvironment,
5835
- withHuggingFaceCacheEnvironment,
5836
- withOfflineHuggingFaceCacheEnvironment,
5837
6159
  tokenF1,
5838
6160
  verifyModelPrefetchCacheReport,
5839
6161
  buildModelPrefetchPayload,
@@ -5856,4 +6178,4 @@ export {
5856
6178
  formatAgentHostBlock,
5857
6179
  assessHardware
5858
6180
  };
5859
- //# sourceMappingURL=chunk-VVKC563R.js.map
6181
+ //# sourceMappingURL=chunk-KRKISPHG.js.map