@tuned-tensor/cli 0.15.0 → 0.16.0

package/dist/{chunk-YGN26POJ.js → chunk-6RLNBTM6.js}

@@ -1076,7 +1076,7 @@ var comparisonReportSchema = z.object({
   }).strict())
 }).strict();
 var trainingReportSchema = z.object({
-  provider: z.literal("local-uv"),
+  provider: z.enum(["local-uv", "aws-ssh"]),
   training_job_name: z.string(),
   model_artifact_uri: z.string().optional(),
   base_model_artifact_uri: z.string().optional(),
@@ -1167,6 +1167,16 @@ var evaluationConfigSchema = z.object({
   }).strict().optional()
 }).strict();
 var localRunnerConfigSchema = z.object({
+  gpu: z.object({
+    provider: z.literal("aws"),
+    instanceId: z.string().regex(/^i-[a-f0-9]{8,17}$/),
+    profile: z.string().min(1).optional(),
+    region: z.string().min(1).optional(),
+    user: z.string().regex(/^[a-z_][a-z0-9_-]*$/i),
+    identityFile: z.string().min(1).optional(),
+    privateIp: z.boolean().default(false),
+    maxSeconds: z.number().int().min(60).max(604800).default(86400)
+  }).strict().optional(),
   storeRoot: z.string().optional(),
   artifactRoot: z.string().default(DEFAULT_ARTIFACT_ROOT),
   dryRun: z.boolean().default(false),
@@ -2907,17 +2917,240 @@ function evaluateGeneralRegressionGate(comparison, policy) {
 }
 
 // src/local-runtime/orchestrator.ts
-import { lstat as lstat5, mkdir as mkdir8, open as open3, readFile as readFile8, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
+import { lstat as lstat5, mkdir as mkdir9, open as open3, readFile as readFile8, readdir as readdir5, rename as rename3, rm as rm4, stat as stat7 } from "fs/promises";
 import { createReadStream as createReadStream3 } from "fs";
-import { createHash as createHash6, randomUUID as randomUUID4 } from "crypto";
+import { createHash as createHash7, randomUUID as randomUUID5 } from "crypto";
 import { performance } from "perf_hooks";
 import { homedir as homedir4 } from "os";
-import { basename, dirname as dirname8, join as join9, relative as relative4, resolve as resolve12 } from "path";
+import { basename as basename2, dirname as dirname9, join as join10, relative as relative4, resolve as resolve13 } from "path";
+
+// src/local-runtime/gpu-executor.ts
+import { createHash as createHash5, randomUUID as randomUUID4 } from "crypto";
+import { mkdtemp, mkdir as mkdir6, realpath as realpath3, rm as rm2, stat as stat5, writeFile as writeFile4 } from "fs/promises";
+import { tmpdir } from "os";
+import { basename, dirname as dirname7, join as join7, resolve as resolve10 } from "path";
+function shellQuote(value) {
+  return `'${value.replaceAll("'", "'\\''")}'`;
+}
+async function resolveGpuHost(gpu) {
+  let stdout = "";
+  const result = await runLoggedProcess({
+    command: "aws",
+    commandArgs: [
+      "ec2",
+      "describe-instances",
+      "--instance-ids",
+      gpu.instanceId,
+      ...gpu.profile ? ["--profile", gpu.profile] : [],
+      ...gpu.region ? ["--region", gpu.region] : [],
+      "--output",
+      "json",
+      "--no-cli-pager"
+    ],
+    timeoutMs: 3e4,
+    stage: "gpu_connect",
+    onLine: (line, stream) => {
+      if (stream === "stdout") stdout += line;
+    }
+  });
+  if (result.exitCode !== 0) throw new Error(`AWS GPU lookup failed: ${result.stderr}`);
+  const instances = JSON.parse(stdout).Reservations?.flatMap((r) => r.Instances ?? []);
+  const instance = instances?.find((i) => i.InstanceId === gpu.instanceId);
+  if (instance?.State?.Name !== "running") throw new Error(`AWS GPU instance ${gpu.instanceId} must already be running.`);
+  const host = gpu.privateIp ? instance.PrivateIpAddress : instance.PublicIpAddress;
+  if (typeof host !== "string" || !/^[0-9.]+$/.test(host)) {
+    throw new Error("AWS GPU instance has no selected IP address. Use privateIp with a reachable VPC/VPN, or assign a public IP.");
+  }
+  return `${gpu.user}@${host}`;
+}
+function sshOptions(gpu) {
+  return [
+    "-o",
+    "BatchMode=yes",
+    "-o",
+    "StrictHostKeyChecking=yes",
+    "-o",
+    "ConnectTimeout=15",
+    "-o",
+    "ServerAliveInterval=15",
+    "-o",
+    "ServerAliveCountMax=3",
+    ...gpu.identityFile ? ["-i", gpu.identityFile] : []
+  ];
+}
+async function checkAwsGpu(gpu) {
+  const host = await resolveGpuHost(gpu);
+  const result = await runLoggedProcess({
+    command: "ssh",
+    commandArgs: [
+      ...sshOptions(gpu),
+      host,
+      `bash -lc ${shellQuote("command -v uv && command -v rsync && command -v timeout && command -v setsid && nvidia-smi")}`
+    ],
+    stage: "gpu_check",
+    timeoutMs: 3e4
+  });
+  if (result.exitCode !== 0) throw new Error(`AWS GPU preflight failed: ${result.stderr}`);
+}
+function gpuBaseModelPath(config, model, revision) {
+  if (config.paths.baseModel) return resolve10(config.paths.baseModel);
+  if (!revision || !/^[a-f0-9]{40}$/i.test(revision)) {
+    throw new Error("AWS GPU execution needs an immutable base-model revision. Run tt models prefetch first or configure paths.baseModel.");
+  }
+  const env = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
+  return join7(env.HF_HUB_CACHE, `models--${model.replaceAll("/", "--")}`, "snapshots", revision);
+}
+async function physicalTransferPath(path) {
+  try {
+    return await realpath3(path);
+  } catch (error) {
+    if (error.code !== "ENOENT" || dirname7(path) === path) throw error;
+    return join7(await physicalTransferPath(dirname7(path)), basename(path));
+  }
+}
+async function assertSeparateTransferPaths(files) {
+  const paths = await Promise.all(files.map((file) => physicalTransferPath(resolve10(file.path))));
+  for (let i = 0; i < files.length; i += 1) {
+    for (let j = 0; j < i; j += 1) {
+      if (files[i].direction === "input" && files[j].direction === "input") continue;
+      const left = paths[i];
+      const right = paths[j];
+      if (left === right || files[i].directory && right.startsWith(`${left}/`) || files[j].directory && left.startsWith(`${right}/`)) {
+        throw new Error(`GPU transfer paths overlap: ${files[i].path} and ${files[j].path}. Use separate input, output, recovery, and backup directories.`);
+      }
+    }
+  }
+}
+async function runGpuProcess(args) {
+  if (await args.shouldCancel?.()) throw new ProcessCancelledError();
+  await assertSeparateTransferPaths(args.files);
+  const host = await resolveGpuHost(args.gpu);
+  const options = sshOptions(args.gpu);
+  const identity = args.logPath ? createHash5("sha256").update(resolve10(args.logPath)).digest("hex").slice(0, 32) : randomUUID4();
+  const root = `/tmp/tt-gpu-${identity}`;
+  const scratch = await mkdtemp(join7(tmpdir(), "tt-gpu-"));
+  const runtime = join7(localRuntimePackageRoot(import.meta.url), "training", args.runtime);
+  const mappings = args.files.map((file, index) => ({ ...file, path: resolve10(file.path), remote: `${root}/files/${index}/${basename(file.path)}` }));
+  const mapPath = (path) => {
+    const absolute = resolve10(path);
+    const match = [...mappings].sort((a, b) => b.path.length - a.path.length).find((file) => absolute === file.path || file.directory && absolute.startsWith(`${file.path}/`));
+    if (match) return match.remote + absolute.slice(match.path.length);
+    if (absolute === runtime || absolute.startsWith(`${runtime}/`)) return `${root}/runtime${absolute.slice(runtime.length)}`;
+    throw new Error(`GPU process path is not declared for transfer: ${path}`);
+  };
+  const transport = async (command, commandArgs, cancellable = true) => {
+    const result = await runLoggedProcess({
+      command,
+      commandArgs,
+      stage: args.stage,
+      reporter: args.reporter,
+      timeoutMs: 30 * 6e4,
+      shouldCancel: cancellable ? args.shouldCancel : void 0
+    });
+    if (result.exitCode !== 0) throw new Error(`${command} failed for ${host}: ${result.stderr}`);
+  };
+  const ssh = async (script, cancellable = true) => transport("ssh", [...options, host, script], cancellable);
+  const sync = async (source, destination, upload, cancellable = true, mirror = false) => {
+    await transport("rsync", [
+      "-rlt",
+      "--copy-links",
+      "--protect-args",
+      "--modify-window=-1",
+      "--chmod=Du=rwx,Dgo=,Fu=rw,Fgo=",
+      ...!upload ? ["--delay-updates", ...mirror ? ["--delete-delay"] : []] : [],
+      "--exclude=.venv",
+      "--exclude=__pycache__",
+      "--exclude=.git",
+      "-e",
+      ["ssh", ...options].map(shellQuote).join(" "),
+      "--",
+      upload ? source : `${host}:${source}`,
+      upload ? `${host}:${destination}` : destination
+    ], cancellable);
+  };
+  const grace = Math.ceil((args.shutdownGraceMs ?? 5e3) / 1e3);
+  const stop = `kill -TERM -- -"$pid" 2>/dev/null || true; remaining=${grace}; while kill -0 -- -"$pid" 2>/dev/null && [ "$remaining" -gt 0 ]; do sleep 1; remaining=$((remaining - 1)); done; kill -KILL -- -"$pid" 2>/dev/null || true`;
+  const owner = randomUUID4();
+  let reservationAttempted = false;
+  let started = false;
+  let completed = false;
+  try {
+    reservationAttempted = true;
+    await ssh(`umask 077; mkdir ${root} && printf '%s' ${shellQuote(owner)} > ${root}/.owner && mkdir ${root}/runtime ${root}/files`).catch((error) => {
+      if (error instanceof ProcessCancelledError) throw error;
+      throw new Error(`Cannot reserve GPU staging directory ${host}:${root}. Another process or unrecovered artifacts may be present. ${error instanceof Error ? error.message : error}`);
+    });
+    for (const name of ["pyproject.toml", "uv.lock", "src"]) {
+      await sync(join7(runtime, name), `${root}/runtime/`, true);
+    }
+    for (const file of mappings) {
+      await ssh(`mkdir -p ${shellQuote(file.directory ? file.remote : dirname7(file.remote))}`);
+      if (file.direction !== "output") {
+        const exists3 = await stat5(file.path).catch((error) => {
+          if (error.code === "ENOENT" && file.direction === "both") return void 0;
+          throw error;
+        });
+        if (exists3) await sync(file.path + (file.directory ? "/" : ""), file.remote + (file.directory ? "/" : ""), true);
+      }
+    }
+    if (args.document) {
+      const value = { ...args.document.value };
+      for (const key of args.document.pathKeys) if (typeof value[key] === "string") value[key] = mapPath(value[key]);
+      const path = join7(scratch, "document.json");
+      await writeFile4(path, JSON.stringify(value), { mode: 384 });
+      await sync(path, mapPath(args.document.path), true);
+    }
+    const env = {
+      UV_PROJECT_ENVIRONMENT: `${root}/venv`,
+      TORCH_DISABLE_NATIVE_JIT: "1",
+      HF_HUB_OFFLINE: "1",
+      TRANSFORMERS_OFFLINE: "1"
+    };
+    for (const [key, value] of Object.entries(args.env ?? {})) {
+      if (value === void 0) continue;
+      if (key === "BACKEND") env[key] = value;
+      if (["SM_CHANNEL_TRAINING", "SM_CHANNEL_BASE_MODEL", "SM_MODEL_DIR", "SM_OUTPUT_DIR", "TT_HYPERPARAMETERS_PATH"].includes(key)) env[key] = mapPath(value);
+    }
+    const commandArgs = args.commandArgs.map((value) => value.startsWith("/") ? mapPath(value) : value);
+    const command = ["env", ...Object.entries(env).map(([key, value]) => `${key}=${value}`), args.command, ...commandArgs].map(shellQuote).join(" ");
+    const script = `umask 077; cd ${root}; setsid timeout -k 120 ${args.gpu.maxSeconds}s ${command} & pid=$!; echo "$pid" > pid; trap ${shellQuote(stop)} HUP INT TERM; wait "$pid"`;
+    await args.reporter?.onEvent?.({ stage: args.stage, status: "running", message: `Running GPU process on ${args.gpu.instanceId}.`, details: { remote_directory: root } });
+    started = true;
+    const result = await runLoggedProcess({
+      ...args,
+      env: process.env,
+      command: "ssh",
+      commandArgs: [...options, "-tt", host, `bash -lc ${shellQuote(script)}`]
+    });
+    completed = result.exitCode === 0;
+    return result;
+  } finally {
+    try {
+      if (started) {
+        await ssh(`bash -c ${shellQuote(`if test -f ${root}/pid; then pid=$(cat ${root}/pid); ${stop}; fi`)}`, false);
+        for (const file of mappings.filter((file2) => file2.direction !== "input")) {
+          await mkdir6(file.directory ? file.path : dirname7(file.path), { recursive: true });
+          if (!completed && !file.directory) {
+            const exists3 = await runLoggedProcess({ command: "ssh", commandArgs: [...options, host, `test -e ${shellQuote(file.remote)}`], stage: args.stage, timeoutMs: 3e4 });
+            if (exists3.exitCode === 1) continue;
+            if (exists3.exitCode !== 0) throw new Error("Cannot inspect remote outputs.");
+          }
+          await sync(file.remote + (file.directory ? "/" : ""), file.path + (file.directory ? "/" : ""), false, false, file.directory === true && file.direction === "both");
+        }
+      }
+      if (reservationAttempted) await ssh(`if [ "$(cat ${root}/.owner 2>/dev/null)" = ${shellQuote(owner)} ]; then rm -rf -- ${root}; fi`, false);
+    } catch (error) {
+      throw new Error(`GPU cleanup or artifact retrieval failed. Remote files remain at ${host}:${root}. ${error instanceof Error ? error.message : error}`);
+    } finally {
+      await rm2(scratch, { recursive: true, force: true });
+    }
+  }
+}
 
 // src/local-runtime/evaluation.ts
-import { createHash as createHash5 } from "crypto";
-import { mkdir as mkdir6, readFile as readFile6, rm as rm2, writeFile as writeFile4 } from "fs/promises";
-import { dirname as dirname7, join as join7, resolve as resolve10 } from "path";
+import { createHash as createHash6 } from "crypto";
+import { mkdir as mkdir7, readFile as readFile6, rm as rm3, stat as stat6, writeFile as writeFile5 } from "fs/promises";
+import { dirname as dirname8, join as join8, resolve as resolve11 } from "path";
 function normalize(value) {
   return value.trim().replace(/\s+/g, " ").toLowerCase();
 }
@@ -3137,8 +3370,8 @@ async function runTransformersInference(args) {
   const inputPath = `${args.outputPath}.inference-input.json`;
   const outputPath = `${args.outputPath}.inference-output.json`;
   const logPath = `${args.outputPath}.inference.log`;
-  await mkdir6(dirname7(inputPath), { recursive: true });
-  await writeFile4(inputPath, `${JSON.stringify({
+  await mkdir7(dirname8(inputPath), { recursive: true });
+  await writeFile5(inputPath, `${JSON.stringify({
     protocol_version: INFERENCE_PROTOCOL_VERSION,
     kind: args.kind,
     model_id: args.modelId,
@@ -3152,7 +3385,7 @@ async function runTransformersInference(args) {
       id: String(index),
       input: example.input
     })),
-    model_cache: args.config.paths.modelCache ? resolve10(args.config.paths.modelCache) : void 0,
+    model_cache: args.config.paths.modelCache ? resolve11(args.config.paths.modelCache) : void 0,
     trust_remote_code: false,
     device: args.config.evaluation.inference.device,
     generation: {
@@ -3162,7 +3395,7 @@ async function runTransformersInference(args) {
     }
   }, null, 2)}
 `, "utf8");
-  await rm2(outputPath, { force: true });
+  await rm3(outputPath, { force: true });
   const entrypoint = buildBundledPythonCommand(
     "evaluate.py",
     ["--input", inputPath, "--output", outputPath]
@@ -3178,7 +3411,30 @@ async function runTransformersInference(args) {
       log_path: logPath
     }
   });
-  const result = await runLoggedProcess({
+  const gpu = args.config.evaluation.inference.device === "cuda" ? args.config.gpu : void 0;
+  const execute = gpu ? async (processArgs) => {
+    const document = JSON.parse(await readFile6(inputPath, "utf8"));
+    document.model_source = gpuBaseModelPath(args.config, args.baseModelId, args.baseModelRevision);
+    delete document.model_cache;
+    const files = [
+      { path: inputPath, direction: "input" },
+      { path: outputPath, direction: "output" },
+      { path: document.model_source, direction: "input", directory: true },
+      ...document.adapter_path ? [{
+        path: document.adapter_path,
+        direction: "input",
+        directory: (await stat6(document.adapter_path)).isDirectory()
+      }] : []
+    ];
+    return runGpuProcess({
+      ...processArgs,
+      gpu,
+      runtime: "adapter",
+      files,
+      document: { path: inputPath, value: document, pathKeys: ["model_source", "adapter_path"] }
+    });
+  } : runLoggedProcess;
+  const result = await execute({
     command: entrypoint.command,
     commandArgs: entrypoint.commandArgs,
     env: withBundledPythonEnvironment(
@@ -3218,7 +3474,7 @@ function assertEvaluationScoringReady(config) {
   }
 }
 function baselineCacheKey(args) {
-  return createHash5("sha256").update(JSON.stringify({
+  return createHash6("sha256").update(JSON.stringify({
     v: 6,
     inference_protocol_version: INFERENCE_PROTOCOL_VERSION,
     package_version: args.packageVersion,
@@ -3238,10 +3494,10 @@ function baselineInputsAreStable(args) {
   return args.config.paths.baseModel ? Boolean(args.sourceFingerprint) : Boolean(args.baseModelRevision);
 }
 function baselineStoreRoot(config) {
-  return config.storeRoot ? resolve10(config.storeRoot) : defaultLocalHome();
+  return config.storeRoot ? resolve11(config.storeRoot) : defaultLocalHome();
 }
 function baselineCachePath(config, key) {
-  return join7(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
+  return join8(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
 }
 async function readBaselineCache(config, path) {
   await hardenExistingLocalStore(baselineStoreRoot(config));
@@ -3255,7 +3511,7 @@ var cachedPackageVersion = null;
 async function packageVersion() {
   if (cachedPackageVersion !== null) return cachedPackageVersion;
   try {
-    const raw = await readFile6(resolve10(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
+    const raw = await readFile6(resolve11(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
     cachedPackageVersion = String(
       JSON.parse(raw).version ?? "unknown"
     );
@@ -3324,7 +3580,7 @@ async function evaluateExamples(args) {
     baseModelId,
     args.baseModelRevision
   );
-  const localModelSource = args.modelSource ? resolve10(args.modelSource) !== resolve10(baseModelId) : false;
+  const localModelSource = args.modelSource ? resolve11(args.modelSource) !== resolve11(baseModelId) : false;
   const baseModelRevision = localModelSource ? defaultBaseModelRevision(baseModelId) : requestedBaseModelRevision;
   const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
   const truncated = args.examples.length > maxExamples;
@@ -3440,8 +3696,8 @@ function compareEvalReports(baseline, candidate) {
 }
 
 // src/local-runtime/process-training.ts
-import { access, mkdir as mkdir7, readFile as readFile7, writeFile as writeFile5 } from "fs/promises";
-import { join as join8, resolve as resolve11 } from "path";
+import { access, mkdir as mkdir8, readFile as readFile7, writeFile as writeFile6 } from "fs/promises";
+import { join as join9, resolve as resolve12 } from "path";
 function buildTrainingHyperparameters(request, options = {}) {
   const model = resolveTrainingModel(request.spec_snapshot.base_model);
   const hyper = request.hyperparameters;
@@ -3532,15 +3788,15 @@ function createTrainingProgressForwarder(reporter) {
 async function launchProcessTraining(args) {
   const { request, artifacts, config } = args;
   const jobName = `tt-local-${request.run_id}`;
-  await mkdir7(artifacts.trainingDir, { recursive: true });
+  await mkdir8(artifacts.trainingDir, { recursive: true });
   await copyDatasetToTrainingChannel(artifacts);
   const hyperparameters = buildTrainingHyperparameters(request, {
     baseModelRevision: args.baseModelRevision
   });
-  await writeJson(join8(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
+  await writeJson(join9(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
   const entrypoint = buildBundledPythonCommand("train.py");
   if (config.dryRun) {
-    await writeFile5(
+    await writeFile6(
       artifacts.trainingLog,
       "Dry run enabled. Training process was not launched.\n",
       "utf8"
@@ -3562,20 +3818,22 @@ async function launchProcessTraining(args) {
     withOfflineHuggingFaceCacheEnvironment({
       ...inheritedEnv,
       BACKEND: "local",
-      SM_CHANNEL_TRAINING: resolve11(artifacts.trainingInputDir),
-      SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve11(config.paths.baseModel) : void 0,
-      SM_MODEL_DIR: resolve11(artifacts.trainingModelDir),
-      SM_OUTPUT_DIR: resolve11(artifacts.trainingOutputDir),
-      TT_HYPERPARAMETERS_PATH: resolve11(artifacts.trainingConfigDir, "hyperparameters.json")
+      SM_CHANNEL_TRAINING: resolve12(artifacts.trainingInputDir),
+      SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve12(config.paths.baseModel) : void 0,
+      SM_MODEL_DIR: resolve12(artifacts.trainingModelDir),
+      SM_OUTPUT_DIR: resolve12(artifacts.trainingOutputDir),
+      TT_HYPERPARAMETERS_PATH: resolve12(artifacts.trainingConfigDir, "hyperparameters.json")
     }, config.paths.modelCache)
   );
   for (const [key, value] of Object.entries(childEnv)) {
     if (value === void 0) delete childEnv[key];
   }
-  if (config.paths.modelCache) await mkdir7(resolve11(config.paths.modelCache), { recursive: true });
+  const remoteBaseModel = config.gpu ? gpuBaseModelPath(config, request.spec_snapshot.base_model, args.baseModelRevision) : void 0;
+  if (remoteBaseModel) childEnv.SM_CHANNEL_BASE_MODEL = remoteBaseModel;
+  if (config.paths.modelCache) await mkdir8(resolve12(config.paths.modelCache), { recursive: true });
   if (config.paths.baseModel) {
     await verifyLocalBaseModel(
-      resolve11(config.paths.baseModel),
+      resolve12(config.paths.baseModel),
       request.spec_snapshot.base_model,
       config.paths.modelCache
     );
@@ -3592,7 +3850,19 @@ async function launchProcessTraining(args) {
     }
   });
   const forwardTrainingProgress = createTrainingProgressForwarder(args.reporter);
-  const { exitCode } = await runLoggedProcess({
+  const execute = config.gpu ? (processArgs) => runGpuProcess({
+    ...processArgs,
+    gpu: config.gpu,
+    runtime: "adapter",
+    files: [
+      { path: artifacts.trainingInputDir, direction: "input", directory: true },
+      { path: artifacts.trainingConfigDir, direction: "input", directory: true },
+      { path: remoteBaseModel, direction: "input", directory: true },
+      { path: artifacts.trainingModelDir, direction: "output", directory: true },
+      { path: artifacts.trainingOutputDir, direction: "output", directory: true }
+    ]
+  }) : runLoggedProcess;
+  const { exitCode } = await execute({
     command: entrypoint.command,
     commandArgs: entrypoint.commandArgs,
     env: childEnv,
@@ -3602,12 +3872,12 @@ async function launchProcessTraining(args) {
     onLine: (line) => forwardTrainingProgress(line),
     shouldCancel: args.shouldCancel
   });
-  const metricsPath = join8(artifacts.trainingModelDir, "training-metrics.json");
+  const metricsPath = join9(artifacts.trainingModelDir, "training-metrics.json");
   const metrics = await pathExists(metricsPath) ? JSON.parse(await readFile7(metricsPath, "utf8")) : null;
   if (exitCode !== 0) {
     throw new Error(`Training process exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
   }
-  const modelTarPath = join8(artifacts.trainingOutputDir, "model.tar.gz");
+  const modelTarPath = join9(artifacts.trainingOutputDir, "model.tar.gz");
   const modelUri = await pathExists(modelTarPath) ? fileUri(modelTarPath) : fileUri(artifacts.trainingModelDir);
   await args.reporter?.onEvent?.({
     stage: "training",
@@ -3621,7 +3891,7 @@ async function launchProcessTraining(args) {
     }
   });
   return {
-    provider: "local-uv",
+    provider: config.gpu ? "aws-ssh" : "local-uv",
     training_job_name: jobName,
     model_artifact_uri: modelUri,
     base_model_artifact_uri: config.paths.baseModel ? fileUri(config.paths.baseModel) : void 0,
@@ -3710,16 +3980,17 @@ async function loadRunRequest(path) {
 function parseLocalRunnerConfig(document, path) {
   const config = localRunnerConfigSchema.parse(document);
   if (!path) return config;
-  const configPath = resolve12(path);
-  const base = dirname8(configPath);
+  const configPath = resolve13(path);
+  const base = dirname9(configPath);
   const configPathValue = (value) => {
     if (!value) return void 0;
     if (value === "~") return homedir4();
-    if (value.startsWith("~/")) return resolve12(homedir4(), value.slice(2));
-    return resolve12(base, value);
+    if (value.startsWith("~/")) return resolve13(homedir4(), value.slice(2));
+    return resolve13(base, value);
   };
   return {
     ...config,
+    gpu: config.gpu ? { ...config.gpu, identityFile: configPathValue(config.gpu.identityFile) } : void 0,
     artifactRoot: configPathValue(config.artifactRoot),
     storeRoot: configPathValue(config.storeRoot),
     paths: {
@@ -3737,7 +4008,7 @@ function parseLocalRunnerConfig(document, path) {
 }
 async function loadLocalRunnerConfig(path) {
   if (!path) return parseLocalRunnerConfig({});
-  const configPath = resolve12(path);
+  const configPath = resolve13(path);
   return parseLocalRunnerConfig(
     await loadJsonFile(configPath),
     configPath
@@ -3752,7 +4023,7 @@ function stripFileUri(path) {
 }
 async function pathExists2(path) {
   try {
-    await stat5(path);
+    await stat7(path);
     return true;
   } catch {
     return false;
@@ -3910,7 +4181,7 @@ function canonicalJson2(value) {
   return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(object[key])}`).join(",")}}`;
 }
 function hashJson(value) {
-  return createHash6("sha256").update(canonicalJson2(value)).digest("hex");
+  return createHash7("sha256").update(canonicalJson2(value)).digest("hex");
 }
 var packageRoot2 = localRuntimePackageRoot(import.meta.url);
 async function hashFileIfPresent(path) {
@@ -3922,21 +4193,21 @@ async function hashFileIfPresent(path) {
 }
 async function packageVersion2() {
   try {
-    const parsed = JSON.parse(await readFile8(resolve12(packageRoot2, "package.json"), "utf8"));
+    const parsed = JSON.parse(await readFile8(resolve13(packageRoot2, "package.json"), "utf8"));
     return typeof parsed.version === "string" ? parsed.version : "unknown";
   } catch {
     return "unknown";
   }
 }
 async function runtimeFingerprint() {
-  const bundledProject2 = resolve12(packageRoot2, "training/adapter");
+  const bundledProject2 = resolve13(packageRoot2, "training/adapter");
   return hashJson({
     tt_local_version: await packageVersion2(),
     node_version: process.version,
     platform: process.platform,
     architecture: process.arch,
-    runner_pyproject: await hashFileIfPresent(resolve12(bundledProject2, "pyproject.toml")),
-    runner_uv_lock: await hashFileIfPresent(resolve12(bundledProject2, "uv.lock"))
+    runner_pyproject: await hashFileIfPresent(resolve13(bundledProject2, "pyproject.toml")),
+    runner_uv_lock: await hashFileIfPresent(resolve13(bundledProject2, "uv.lock"))
   });
 }
 function preparedSourceFingerprint(args) {
@@ -3950,7 +4221,7 @@ function preparedSourceFingerprint(args) {
   });
 }
 async function hashFile(path) {
-  const hash = createHash6("sha256");
+  const hash = createHash7("sha256");
   for await (const chunk of createReadStream3(path)) hash.update(chunk);
   return hash.digest("hex");
 }
@@ -3983,7 +4254,7 @@ async function resolveBaseModelRevision(request, config) {
   }
   const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
   const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
-  const refPath = resolve12(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
+  const refPath = resolve13(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
   try {
     const revision = (await readFile8(refPath, "utf8")).trim();
     if (revision) return revision;
@@ -4004,12 +4275,12 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
   const files = [];
   const visit = async (directory) => {
     for (const entry of await readdir5(directory, { withFileTypes: true })) {
-      const child = resolve12(directory, entry.name);
+      const child = resolve13(directory, entry.name);
       if (entry.isDirectory()) {
         await visit(child);
         continue;
       }
-      const childMetadata = await stat5(child);
+      const childMetadata = await stat7(child);
       if (entry.isSymbolicLink() && !childMetadata.isFile()) {
         throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
       }
@@ -4043,27 +4314,27 @@ async function readStageMetadata(path) {
   }
 }
 async function removePrefixedArtifacts(path) {
-  const directory = dirname8(path);
-  const prefix = `${basename(path)}.`;
+  const directory = dirname9(path);
+  const prefix = `${basename2(path)}.`;
   const names = await readdir5(directory).catch(() => []);
   await Promise.all([
-    rm3(path, { force: true }),
-    ...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(resolve12(directory, name), { recursive: true, force: true }))
+    rm4(path, { force: true }),
+    ...names.filter((name) => name.startsWith(prefix)).map((name) => rm4(resolve13(directory, name), { recursive: true, force: true }))
   ]);
 }
 async function cleanupStageArtifacts(artifacts, stage) {
-  const removeReport = async () => rm3(artifacts.runReportJson, { force: true });
+  const removeReport = async () => rm4(artifacts.runReportJson, { force: true });
   if (stage === "prepare") {
     await Promise.all([
       removePrefixedArtifacts(artifacts.baselineEvalJson),
       removePrefixedArtifacts(artifacts.candidateEvalJson),
       removePrefixedArtifacts(artifacts.generalBaselineEvalJson),
       removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
-      rm3(artifacts.trainingDir, { recursive: true, force: true }),
+      rm4(artifacts.trainingDir, { recursive: true, force: true }),
       removePrefixedArtifacts(artifacts.trainingReportJson),
-      rm3(resolve12(artifacts.runDir, "model.tar.gz"), { force: true }),
+      rm4(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
       removeReport(),
-      rm3(artifacts.artifactManifestJson, { force: true })
+      rm4(artifacts.artifactManifestJson, { force: true })
     ]);
     return;
   }
@@ -4077,9 +4348,9 @@ async function cleanupStageArtifacts(artifacts, stage) {
   }
   if (stage === "train") {
     await Promise.all([
-      rm3(artifacts.trainingDir, { recursive: true, force: true }),
+      rm4(artifacts.trainingDir, { recursive: true, force: true }),
       removePrefixedArtifacts(artifacts.trainingReportJson),
-      rm3(resolve12(artifacts.runDir, "model.tar.gz"), { force: true }),
+      rm4(resolve13(artifacts.runDir, "model.tar.gz"), { force: true }),
       removePrefixedArtifacts(artifacts.candidateEvalJson),
       removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
       removeReport()
@@ -4115,8 +4386,8 @@ async function throwIfCancelled(store, request) {
   }
 }
 async function acquireWorkflowLock(lockPath, description) {
-  await mkdir8(dirname8(lockPath), { recursive: true });
-  const token = randomUUID4();
+  await mkdir9(dirname9(lockPath), { recursive: true });
+  const token = randomUUID5();
   for (let attempt = 0; attempt < 3; attempt += 1) {
     try {
       const handle = await open3(lockPath, "wx", 384);
@@ -4129,13 +4400,13 @@ async function acquireWorkflowLock(lockPath, description) {
       return async () => {
         try {
           const owner = JSON.parse(await readFile8(lockPath, "utf8"));
-          if (owner.token === token) await rm3(lockPath, { force: true });
+          if (owner.token === token) await rm4(lockPath, { force: true });
         } catch {
         }
       };
     } catch (error) {
       if (error.code !== "EEXIST") throw error;
-      const metadata = await stat5(lockPath).catch(() => null);
+      const metadata = await stat7(lockPath).catch(() => null);
       let owner = null;
       try {
         owner = JSON.parse(await readFile8(lockPath, "utf8"));
@@ -4157,10 +4428,10 @@ async function acquireWorkflowLock(lockPath, description) {
           `${description} already has an active local workflow (lock: ${lockPath}).`
         );
       }
-      const stalePath = `${lockPath}.stale.${randomUUID4()}`;
+      const stalePath = `${lockPath}.stale.${randomUUID5()}`;
       try {
         await rename3(lockPath, stalePath);
-        await rm3(stalePath, { force: true });
+        await rm4(stalePath, { force: true });
       } catch (renameError) {
         if (renameError.code !== "ENOENT") throw renameError;
       }
@@ -4170,11 +4441,11 @@ async function acquireWorkflowLock(lockPath, description) {
 }
 async function acquireRunLock(store, runId) {
   await store.ensure();
-  return acquireWorkflowLock(resolve12(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
+  return acquireWorkflowLock(resolve13(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
 }
 async function acquireArtifactLock(artifacts) {
   return acquireWorkflowLock(
-    join9(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
+    join10(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
     `Artifact directory ${artifacts.runDir}`
   );
 }
@@ -4221,13 +4492,13 @@ function stageFingerprintPath(prepared, stage) {
   return `${stageOutputPath(prepared, stage)}.stage.json`;
 }
 async function runnerFileFingerprints() {
-  const project = resolve12(packageRoot2, "training/adapter");
-  const bundledSource = resolve12(packageRoot2, "training/adapter/src");
-  const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve12(bundledSource, entry.name));
+  const project = resolve13(packageRoot2, "training/adapter");
+  const bundledSource = resolve13(packageRoot2, "training/adapter/src");
+  const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve13(bundledSource, entry.name));
   const candidates = [.../* @__PURE__ */ new Set([
     ...pythonSources,
-    resolve12(project, "pyproject.toml"),
-    resolve12(project, "uv.lock")
+    resolve13(project, "pyproject.toml"),
+    resolve13(project, "uv.lock")
   ])];
   const fingerprints = {};
   for (const path of candidates.sort()) fingerprints[path] = await hashFileIfPresent(path);
@@ -4341,7 +4612,7 @@ async function ensureRunRecord(args) {
         `Run ${args.request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
       );
     }
-    if (resolve12(existing.artifact_dir) !== resolve12(args.artifacts.runDir)) {
+    if (resolve13(existing.artifact_dir) !== resolve13(args.artifacts.runDir)) {
       throw new Error(
         `Run ${args.request.run_id} is already bound to artifact directory ${existing.artifact_dir}; it cannot be resumed with ${args.artifacts.runDir}.`
       );
@@ -4419,7 +4690,7 @@ async function computePreparedRun(args) {
     base_model_for_evaluation: baseModelForEvaluation,
     base_model_revision: baseModelRevision ?? null,
     base_model_fingerprint: baseModelFingerprint ?? null,
-    system_prompt_sha256: createHash6("sha256").update(system).digest("hex"),
+    system_prompt_sha256: createHash7("sha256").update(system).digest("hex"),
     prepared_at: (/* @__PURE__ */ new Date()).toISOString()
   };
   if (args.writeArtifacts) await writeJsonAtomic(artifacts.stageMetadataJson, metadata);
@@ -4448,16 +4719,24 @@ async function prepareStage(args) {
   const preparedExists = await pathExists2(args.artifacts.stageMetadataJson) && await pathExists2(args.artifacts.trainingJsonl);
   const prepared = await computePreparedRun({ ...args, writeArtifacts: false });
   await args.store.syncRunRequest(args.request, args.artifacts.runDir);
-  await writeJsonAtomic(resolve12(args.artifacts.runDir, "request.json"), args.request);
+  await writeJsonAtomic(resolve13(args.artifacts.runDir, "request.json"), args.request);
   const existingMetadata = preparedExists ? await readStageMetadata(args.artifacts.stageMetadataJson) : null;
   let canReuse = preparedExists && existingMetadata?.source_fingerprint === prepared.metadata.source_fingerprint;
   if (canReuse) {
     canReuse = await verifyReusableArtifacts(args.artifacts, [
       args.artifacts.stageMetadataJson,
       args.artifacts.trainingJsonl
-    ]);
+    ], { verifyModel: true });
   }
   if (canReuse) {
+    for (const stage of ["baseline", "train", "candidate"]) {
+      await canReuseStageArtifact({
+        stage,
+        prepared,
+        config: args.config,
+        additionalPaths: stage === "train" ? [] : generalRegressionRequiredPaths(prepared, stage)
+      });
+    }
     await throwIfCancelled(args.store, args.request);
     await updateRun({
       ...args,
@@ -4609,8 +4888,8 @@ async function runTrainStage(args) {
     request: args.prepared.request,
     status: "training",
     stage: "training",
-    message: args.config.dryRun ? "Recording dry-run training result." : "Launching local training process.",
-    details: { training_backend: "local-uv", dry_run: args.config.dryRun }
+    message: args.config.dryRun ? "Recording dry-run training result." : args.config.gpu ? "Launching training on your AWS GPU." : "Launching local training process.",
+    details: { training_backend: args.config.gpu && !args.config.dryRun ? "aws-ssh" : "local-uv", dry_run: args.config.dryRun }
   });
   const training = await launchProcessTraining({
     request: args.prepared.request,
@@ -4977,7 +5256,7 @@ async function runLocalPipeline(input) {
         if (step.uses === "evaluate") {
           const isBase = step.with.model === "base";
           const report = isBase ? await runBaselineStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter }) : await runCandidateStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
-          const reportPath = join9(artifacts.runDir, "evaluations", `${step.id}.json`);
+          const reportPath = join10(artifacts.runDir, "evaluations", `${step.id}.json`);
           await writeJsonAtomic(reportPath, { ...report, artifact_uri: fileUri(reportPath) });
           outputs[step.id] = { report: { ...report, artifact_uri: fileUri(reportPath) }, reportPath };
           await refreshArtifactManifest(prepared);
@@ -5298,10 +5577,7 @@ function evaluateCapabilities(inventory) {
     train,
     finetune: train,
     inference: hasCuda ? inference : { ...inference, reason: `CPU eval: ${inference.reason}` },
-    serve: {
-      status: "not_possible",
-      reason: "tt serve cannot host foundation checkpoints yet"
-    }
+    serve: hasCuda && inventory.os.platform === "linux" ? inference : { status: "not_possible", reason: "Foundation serving requires Linux and an NVIDIA CUDA GPU" }
   };
   return { cuda_available: hasCuda, gpu, adapters, foundation, notes };
 }
@@ -5315,7 +5591,7 @@ function formatCapabilitySummary(report) {
   return [
     `GPU ${gpuLabel}, CUDA ${report.cuda_available ? "yes" : "no"}`,
     `Adapter: ${adapterBits.join("; ")}`,
-    `Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve not supported`
+    `Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve ${report.foundation.serve.status}`
   ].join(". ");
 }
 function warningsFromSnapshot(report, target) {
@@ -5352,7 +5628,7 @@ function warningsFromSnapshot(report, target) {
 import { constants as constants2 } from "fs";
 import { access as access2, statfs } from "fs/promises";
 import { cpus, freemem, homedir as homedir5, totalmem, type } from "os";
-import { dirname as dirname9, join as join10, resolve as resolve13 } from "path";
+import { dirname as dirname10, join as join11, resolve as resolve14 } from "path";
 import { spawn as spawn2 } from "child_process";
 async function runCommand(command, args, options = {}) {
   const timeoutMs = options.timeoutMs ?? 1e4;
@@ -5506,7 +5782,7 @@ async function queryNvidiaGpus(env) {
   return { result: table.code === 0 || !last ? table : last, gpus };
 }
 async function diskCheck(name, path) {
-  const resolvedPath = resolve13(path);
+  const resolvedPath = resolve14(path);
   try {
     await access2(resolvedPath, constants2.R_OK);
     const fs = await statfs(resolvedPath);
@@ -5521,7 +5797,7 @@ async function diskCheck(name, path) {
   } catch (error) {
     let probe = resolvedPath;
     for (let i = 0; i < 6; i += 1) {
-      const parent = dirname9(probe);
+      const parent = dirname10(probe);
       if (parent === probe) break;
       probe = parent;
       try {
@@ -5607,10 +5883,10 @@ async function collectHostInventory(options = {}) {
   const cwd = options.cwd ?? process.cwd();
   const nodeVersion = process.versions.node;
   const nodeMajor = Number(nodeVersion.split(".")[0]);
-  const artifactRoot = resolve13(cwd, config?.artifactRoot ?? join10(cwd, DEFAULT_ARTIFACT_ROOT));
-  const storeRoot = resolve13(config?.storeRoot ?? defaultStoreRoot(env));
-  const modelCache = resolve13(
-    config?.paths.modelCache ?? env.HF_HOME ?? join10(homedir5(), ".cache", "huggingface")
+  const artifactRoot = resolve14(cwd, config?.artifactRoot ?? join11(cwd, DEFAULT_ARTIFACT_ROOT));
+  const storeRoot = resolve14(config?.storeRoot ?? defaultStoreRoot(env));
+  const modelCache = resolve14(
+    config?.paths.modelCache ?? env.HF_HOME ?? join11(homedir5(), ".cache", "huggingface")
   );
   const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
     diskCheck("artifact-root", artifactRoot),
@@ -5656,12 +5932,12 @@ async function collectHostInventory(options = {}) {
 }
 
 // src/local-runtime/hardware-snapshot.ts
-import { chmod as chmod2, mkdir as mkdir9, readFile as readFile9, writeFile as writeFile6 } from "fs/promises";
-import { dirname as dirname10, join as join11 } from "path";
+import { chmod as chmod2, mkdir as mkdir10, readFile as readFile9, writeFile as writeFile7 } from "fs/promises";
+import { dirname as dirname11, join as join12 } from "path";
 var HARDWARE_SNAPSHOT_VERSION = 1;
 var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
 function hardwareSnapshotPath(env = process.env) {
-  return join11(getTunedTensorHome(env), "hardware.json");
+  return join12(getTunedTensorHome(env), "hardware.json");
 }
 async function readHardwareSnapshot(env = process.env) {
   const path = hardwareSnapshotPath(env);
@@ -5683,7 +5959,7 @@ async function readHardwareSnapshot(env = process.env) {
 }
 async function writeHardwareSnapshot(snapshot, env = process.env) {
   const path = hardwareSnapshotPath(env);
-  await mkdir9(dirname10(path), { recursive: true, mode: 448 });
+  await mkdir10(dirname11(path), { recursive: true, mode: 448 });
   const gpu = snapshot.capabilities.gpu;
   const record = {
     version: HARDWARE_SNAPSHOT_VERSION,
@@ -5694,7 +5970,7 @@ async function writeHardwareSnapshot(snapshot, env = process.env) {
     capabilities: snapshot.capabilities,
     summary: snapshot.summary ?? formatCapabilitySummary(snapshot.capabilities)
   };
-  await writeFile6(path, `${JSON.stringify(record, null, 2)}
+  await writeFile7(path, `${JSON.stringify(record, null, 2)}
 `, { mode: 384 });
   await chmod2(path, 384);
   return record;
@@ -5817,15 +6093,17 @@ export {
   buildFoundationPythonCommand,
   withFoundationPythonEnvironment,
   runLoggedProcess,
+  minimalMachineLearningEnvironment,
+  withHuggingFaceCacheEnvironment,
+  withOfflineHuggingFaceCacheEnvironment,
+  checkAwsGpu,
+  runGpuProcess,
   isTerminalRunState,
   defaultLocalHome,
   localStorePaths,
   hardenExistingLocalStore,
   writePrivateJsonAtomic,
   createLocalStore,
-  minimalMachineLearningEnvironment,
-  withHuggingFaceCacheEnvironment,
-  withOfflineHuggingFaceCacheEnvironment,
   tokenF1,
   verifyModelPrefetchCacheReport,
   buildModelPrefetchPayload,
@@ -5848,4 +6126,4 @@ export {
   formatAgentHostBlock,
   assessHardware
 };
-//# sourceMappingURL=chunk-YGN26POJ.js.map
+//# sourceMappingURL=chunk-6RLNBTM6.js.map