@tuned-tensor/cli 0.15.1 → 0.16.0

package/dist/index.js

@@ -23,12 +23,13 @@ import {
   parseLocalRunInput,
   parseLocalRunnerConfig,
   readHardwareSnapshot,
+  runGpuProcess,
   runLocalPipeline,
   runLoggedProcess,
   runReportSchema,
   warningsFromSnapshot,
   withFoundationPythonEnvironment
-} from "./chunk-VVKC563R.js";
+} from "./chunk-6RLNBTM6.js";
 
 // src/cli.ts
 import { spawn as spawn4 } from "child_process";
@@ -1971,7 +1972,7 @@ function createTunedTensorTools(api, options = {}) {
         return {
           version: 1,
           engine,
-          scope: { execution: "local-only", cloud_supported: false },
+          scope: { execution: "local", gpu_provider: "aws" },
           canonical: canonicalFoundationPipeline2(),
           optional_rl: {
             enabled_when: "foundation.rl_steps > 0",
@@ -1991,17 +1992,13 @@ function createTunedTensorTools(api, options = {}) {
         return {
           version: 1,
           engine,
-          scope: { execution: "cloud", cli: "tt cloud", access_token_required: true },
-          canonical: canonicalPipeline("cloud"),
+          scope: { execution: "local", gpu_provider: "aws", access_token_required: false },
+          canonical: canonicalPipeline("local"),
           commands: {
-            login: "tt auth login",
-            push: "tt cloud push --file tunedtensor.json",
-            estimate: "tt cloud runs estimate <spec-id>",
-            run: "tt cloud runs start <spec-id>",
-            watch: "tt cloud runs watch <run-id>",
-            report: "tt cloud runs report <run-id>"
+            run: "tt pipeline run --file tunedtensor.pipeline.json --spec tunedtensor.json --config local-runner.json",
+            doctor: "tt doctor --config local-runner.json"
           },
-          note: "Use the spec ID returned by push. Cloud recipes cannot be dispatched by tt pipeline run; cloud runs use the hosted run API."
+          note: "Configure gpu in local-runner.json with provider aws, instanceId, SSH user and optional AWS profile/region/identityFile. The instance must already be running and reachable. GPU processes use it automatically; orchestration, scoring and artifacts stay local. Hosted training is retired."
         };
       }
       return {
@@ -2184,7 +2181,7 @@ Mutation tools only prepare proposals. Never claim a proposed mutation happened.
 Do not request or reveal Tuned Tensor or model-provider credentials. You have no shell, upload, delete, top-up, API-key, watch, or serving tools.`;
 async function systemPrompt(cloudEnabled) {
   const host = formatAgentHostBlock(await readHardwareSnapshot());
-  const account = cloudEnabled ? "Account tools inspect cloud specs, runs, models, balance, transactions, and managed agent usage. Cloud spec mutations only prepare proposals for /approve; real cloud runs require the user to invoke `tt cloud runs start` directly. Never confuse cloud resources with local resources." : "No TT access token is configured, so account and cloud tools are unavailable. The user can run `tt auth login` or /login tunedtensor to enable them.";
+  const account = cloudEnabled ? "Account tools inspect cloud specs, runs, models, balance, transactions, and managed agent usage. Cloud spec mutations only prepare proposals for /approve; training uses `tt pipeline run` with optional user-owned AWS GPU configuration. Hosted training submission is retired. Never confuse cloud resources with local resources." : "No TT access token is configured, so account and cloud tools are unavailable. The user can run `tt auth login` or /login tunedtensor to enable them.";
   return `${SYSTEM_PROMPT}
 ${account}
 You have no general filesystem tools. Workspace-scoped capabilities can prepare one new validated spec folder or a sealed local pipeline dry-run; both still require /approve.
@@ -2762,8 +2759,8 @@ var COMMAND_CATALOG = [
   { path: "cloud label cancel", description: "Cancel a labeling job and release unused credits", group: "Cloud", modes: LOCAL },
   { path: "cloud runs list", description: "List runs", group: "Cloud", modes: LOCAL },
   { path: "cloud runs get", description: "Show run details and eval results", group: "Cloud", modes: LOCAL },
-  { path: "cloud runs estimate", description: "Estimate run cost and duration before starting", group: "Cloud", modes: LOCAL },
-  { path: "cloud runs start", description: "Start a new run for a behaviour spec", group: "Cloud", modes: LOCAL },
+  { path: "cloud runs estimate", description: "Retired: use the local pipeline with your own AWS GPU", group: "Cloud", modes: LOCAL },
+  { path: "cloud runs start", description: "Retired: use the local pipeline with your own AWS GPU", group: "Cloud", modes: LOCAL },
   { path: "cloud runs cancel", description: "Cancel a running run", group: "Cloud", modes: LOCAL },
   { path: "cloud runs watch", description: "Watch a run until it completes", group: "Cloud", modes: LOCAL },
   { path: "cloud runs diagnose", description: "Show live run diagnostics", group: "Cloud", modes: LOCAL },
@@ -5170,7 +5167,7 @@ function renderHardware(value) {
   );
   if (foundation) {
     printDetail([
-      ["Foundation serve", text(record3(foundation.serve)?.reason) ?? "cannot host yet"]
+      ["Foundation serve", text(record3(foundation.serve)?.reason) ?? "unknown"]
     ]);
   }
   const notes = Array.isArray(capabilities?.notes) ? capabilities.notes.filter((note) => typeof note === "string") : [];
@@ -6021,7 +6018,27 @@ function assertFoundationPlanSupported(spec, plan) {
 }
 async function defaultFoundationSpawn(args2) {
   const launched = buildFoundationPythonCommand(args2.entrypoint, ["--config", args2.configPath]);
-  const result2 = await runLoggedProcess({
+  const execute = args2.gpu && args2.entrypoint !== "train_tokenizer.py" ? async (processArgs) => {
+    const document = await readRequiredJsonObject(args2.configPath, "Foundation GPU config");
+    const files = [{ path: args2.configPath, direction: "input" }];
+    const pathKeys = ["tokenizer_dir", "model_dir", "corpus_path", "validation_path", "output_dir", "work_dir", "checkpoint_backup_dir"];
+    for (const key of pathKeys) {
+      if (typeof document[key] !== "string") continue;
+      files.push({
+        path: document[key],
+        direction: ["output_dir", "work_dir", "checkpoint_backup_dir"].includes(key) ? "both" : "input",
+        directory: key.endsWith("_dir") || (await lstat4(document[key])).isDirectory()
+      });
+    }
+    return runGpuProcess({
+      ...processArgs,
+      gpu: args2.gpu,
+      runtime: "foundation",
+      files,
+      document: { path: args2.configPath, value: document, pathKeys }
+    });
+  } : runLoggedProcess;
+  const result2 = await execute({
     command: launched.command,
     commandArgs: launched.commandArgs,
     env: withFoundationPythonEnvironment(process.env),
@@ -6042,6 +6059,9 @@ async function runFoundationPipeline(args2) {
   const outputDir = resolve5(
     args2.outputDir ?? join8(dirname5(resolve5(args2.specPath)), DEFAULT_FOUNDATION_RUNS_DIR, randomUUID6())
   );
+  if (args2.gpu && args2.spec.foundation.checkpoint_backup_dir && localPathsOverlap(outputDir, args2.spec.foundation.checkpoint_backup_dir)) {
+    throw new Error("AWS checkpoint_backup_dir must be separate from the foundation run directory; overlapping backups can overwrite returned checkpoints.");
+  }
   let outputExists = false;
   try {
     const outputInfo = await lstat4(outputDir);
@@ -6203,7 +6223,7 @@ async function runFoundationPipeline(args2) {
     if (!completedStep) {
       await writePrivateTextAtomic(configPath2, `${JSON.stringify(config, null, 2)}
 `);
-      await spawnStep({ entrypoint, configPath: configPath2, logPath, stepId: step.id });
+      await spawnStep({ entrypoint, configPath: configPath2, logPath, stepId: step.id, ...args2.gpu ? { gpu: args2.gpu } : {} });
       upstreamExecuted = true;
     }
     await makePrivateTree(stepDir);
@@ -6439,24 +6459,27 @@ function registerPipelineCommands(parent) {
     if (options.output && options.resume) {
       throw new Error("--output and --resume are mutually exclusive.");
     }
+    const config = await loadLocalRunnerConfig(localConfigPath(options.config, options.spec));
     const document = await resolvePipelineDocument(options);
     const plan = createExecutionPlan(document, { only: parseList(options.only), skip: parseList(options.skip) });
-    const hostWarnings = await hostWarningsForPipeline(document, options.spec);
-    if (options.dryRun) {
+    const hostWarnings = config.gpu ? [] : await hostWarningsForPipeline(document, options.spec);
+    if (options.dryRun || config.dryRun) {
       if (isJsonMode()) {
         return printJson({
           dry_run: true,
           ...plan,
+          ...config.gpu ? { gpu: { provider: config.gpu.provider, instanceId: config.gpu.instanceId, region: config.gpu.region, profile: config.gpu.profile } } : {},
           ...hostWarnings.length ? { host_warnings: hostWarnings } : {}
         });
       }
       console.log("Dry run only \u2014 no execution, artifact transfer, or credit reservation will occur.");
+      if (config.gpu) console.log(`GPU processes: AWS instance ${config.gpu.instanceId}. Orchestration stays local.`);
       return outputPlan(plan, hostWarnings);
     }
     for (const warning of hostWarnings) printWarning(warning);
     const remote = plan.steps.find((step) => step.target !== "local");
     if (remote) {
-      throw new Error(`Step "${remote.id}" targets cloud execution. Pipeline execution requires local targets; use tt cloud runs for hosted execution, or --dry-run to inspect this plan.`);
+      throw new Error(`Step "${remote.id}" targets cloud execution. Pipeline execution requires local targets; set targets to local and configure gpu in local-runner.json to use your AWS instance.`);
     }
     const input = await loadLocalRunInput(resolve6(options.spec));
     if (isParsedFoundationPipeline(document) || input.kind === "foundation-spec") {
@@ -6469,7 +6492,8 @@ function registerPipelineCommands(parent) {
         plan,
         specPath: resolve6(options.spec),
         ...options.output || options.resume ? { outputDir: resolve6(options.resume ?? options.output) } : {},
-        resume: Boolean(options.resume)
+        resume: Boolean(options.resume),
+        gpu: config.gpu
       });
       if (isJsonMode()) return printJson(result3);
       printSuccess(`Foundation pipeline completed. Report: ${result3.report_path}`);
@@ -6479,7 +6503,6 @@ function registerPipelineCommands(parent) {
       throw new Error("--output and --resume are only valid for foundation pipelines.");
     }
     assertLocalRunInputReady(input.request);
-    const config = await loadLocalRunnerConfig(localConfigPath(options.config, options.spec));
     const localPipeline = {
       version: 1,
       ...plan.name ? { name: plan.name } : {},
@@ -7856,7 +7879,7 @@ function registerLabelCommands(parent) {
       `Dataset created: ${data3.name} (${shortId(data3.id)}) \u2014 ${data3.row_count} rows.`
     );
     console.log(
-      `Start a run with \`tt cloud runs start <spec-id> --dataset ${shortId(data3.id)}\`.`
+      "Export the dataset for use with tt pipeline run and optional AWS GPU configuration."
     );
   });
   label.command("cancel").description("Cancel a labeling job and release unused credits").argument("<id>", "Labeling job ID (full UUID or 4+ char prefix)").action(async (id) => {
@@ -7874,15 +7897,6 @@ function registerLabelCommands(parent) {
 
 // src/commands/runs.ts
 import ora from "ora";
-var LONG_EXAMPLE_POLICIES = ["error", "truncate", "skip"];
-function parseLongExamplesPolicy(value) {
-  if (LONG_EXAMPLE_POLICIES.includes(value)) {
-    return value;
-  }
-  throw new Error(
-    `--long-examples must be one of: ${LONG_EXAMPLE_POLICIES.join(", ")}`
-  );
-}
 function formatProgress(run) {
   if (run.progress_pct == null && !run.stage_label) return void 0;
   const label = run.stage_label ?? run.current_stage ?? "Progress";
@@ -7915,43 +7929,6 @@ function formatMinutes(minutes) {
   if (minutes < 60) return `${Math.max(1, Math.round(minutes))}m`;
   return `${(minutes / 60).toFixed(1)}h`;
 }
-function formatCents3(cents) {
-  const dollars = cents / 100;
-  if (Math.abs(dollars) >= 100) return `$${dollars.toFixed(0)}`;
-  return `$${dollars.toFixed(2)}`;
-}
-function formatEstimateRange(estimate) {
-  const duration = estimate.duration;
-  return `${formatMinutes(duration.estimated_minutes)} (${formatMinutes(duration.range_minutes.low)}-${formatMinutes(duration.range_minutes.high)})`;
-}
-function printRunEstimate(estimate) {
-  const billing = estimate.billing;
-  printDetail([
-    ["Estimated Time", formatEstimateRange(estimate)],
-    ["Confidence", estimate.duration.confidence],
-    ["History Samples", String(estimate.duration.sample_count)],
-    ["Basis", estimate.duration.basis.replaceAll("_", " ")],
-    ["Estimated Cost", formatCents3(estimate.estimated_cost_cents)],
-    ["Training Tokens", `${(estimate.estimated_training_tokens / 1e3).toFixed(1)}k`],
-    ["Epochs", String(estimate.estimated_epochs)],
-    ["Plan", billing?.plan],
-    [
-      "Billing Source",
-      billing?.billing_source === "free_quota" ? "Free monthly quota" : billing?.billing_source === "credits" ? "Credits" : void 0
-    ],
-    [
-      "Free Runs",
-      billing ? `${billing.free_runs_remaining}/${billing.free_runs_monthly_limit} remaining` : void 0
-    ],
-    [
-      "Free Eligible",
-      billing ? billing.free_run_eligible ? "yes" : `no (${billing.free_run_ineligibility.join(", ") || "not eligible"})` : void 0
-    ]
-  ]);
-  console.log(
-    "\nDuration is a rough historical range. Final cost uses provider-reported training tokens."
-  );
-}
 function formatEpochProgress(diagnostics) {
   const curve = diagnostics.training.curve;
   if (curve.latest_epoch == null) return void 0;
@@ -8182,46 +8159,6 @@ function printRunReport(report, options) {
     printReportExamples(label, split, options.mode, options.limit);
   }
 }
-function addRunConfigurationOptions(command) {
-  return command.option("--no-augment", "Disable data augmentation").option("--no-llm-judge", "Disable LLM judging").option("--epochs <n>", "Number of training epochs").option("--lr <rate>", "Learning rate").option("--batch-size <n>", "Batch size").option("--dataset <id>", "Dataset ID to use instead of inline spec examples (full UUID or 4+ char prefix)").option("--parent-model <id>", "Fine-tuned model ID to continue training from (full UUID or 4+ char prefix)").option("--train-ratio <ratio>", "Dataset training split ratio (default: 0.8 when any split ratio is set)").option("--validation-ratio <ratio>", "Dataset validation split ratio (default: 0.1 when any split ratio is set)").option("--test-ratio <ratio>", "Dataset test split ratio (default: 0.1 when any split ratio is set)").option("--lora-rank <n>", "LoRA rank").option("--lora-alpha <n>", "LoRA alpha").option("--max-eval-examples <n>", "Max examples for the primary eval pass").option("--max-test-eval-examples <n>", "Max examples for the secondary test eval pass").option("--long-examples <policy>", "Long training row policy: error, truncate, or skip").option("--max-seq-length <tokens>", "Maximum training sequence length in tokens").option("--max-output-tokens <tokens>", "Desired evaluation output budget in tokens").option("--eval-reserved-output-tokens <tokens>", "Minimum evaluation output tokens reserved per row");
-}
-async function buildRunRequestBody(cmdOpts, opts) {
-  const body = {};
-  if (cmdOpts.augment === false) body.augment = false;
-  if (cmdOpts.dataset) body.dataset_id = await resolveDatasetId(String(cmdOpts.dataset), opts);
-  if (cmdOpts.parentModel) {
-    body.parent_model_id = await resolveModelId(String(cmdOpts.parentModel), opts);
-  }
-  const splitRatioOptions = [
-    cmdOpts.trainRatio,
-    cmdOpts.validationRatio,
-    cmdOpts.testRatio
-  ];
-  if (splitRatioOptions.some((value) => value != null)) {
-    body.split_ratios = {
-      train: cmdOpts.trainRatio != null ? Number(cmdOpts.trainRatio) : 0.8,
-      validation: cmdOpts.validationRatio != null ? Number(cmdOpts.validationRatio) : 0.1,
-      test: cmdOpts.testRatio != null ? Number(cmdOpts.testRatio) : 0.1
-    };
-  }
-  const hp = {};
-  if (cmdOpts.epochs) hp.n_epochs = Number(cmdOpts.epochs);
-  if (cmdOpts.lr) hp.learning_rate = Number(cmdOpts.lr);
-  if (cmdOpts.batchSize) hp.batch_size = Number(cmdOpts.batchSize);
-  if (cmdOpts.loraRank) hp.lora_rank = Number(cmdOpts.loraRank);
-  if (cmdOpts.loraAlpha) hp.lora_alpha = Number(cmdOpts.loraAlpha);
-  if (cmdOpts.maxEvalExamples) hp.max_eval_examples = Number(cmdOpts.maxEvalExamples);
-  if (cmdOpts.maxTestEvalExamples) hp.max_test_eval_examples = Number(cmdOpts.maxTestEvalExamples);
-  if (cmdOpts.longExamples) hp.long_examples = parseLongExamplesPolicy(String(cmdOpts.longExamples));
-  if (cmdOpts.maxSeqLength) hp.max_seq_length = Number(cmdOpts.maxSeqLength);
-  if (cmdOpts.maxOutputTokens) hp.max_output_tokens = Number(cmdOpts.maxOutputTokens);
-  if (cmdOpts.evalReservedOutputTokens) {
-    hp.eval_reserved_output_tokens = Number(cmdOpts.evalReservedOutputTokens);
-  }
-  if (cmdOpts.llmJudge === false) body.use_llm_judge = false;
-  if (Object.keys(hp).length) body.hyperparameters = hp;
-  return body;
-}
 function registerRunsCommands(parent) {
   const runs = parent.command("runs").description("Manage runs");
   runs.command("archive").description("Archive an inactive local run report published to the dashboard").argument("<id>", "Published run ID or prefix").action(async (id) => {
@@ -8312,36 +8249,11 @@ Eval Results (${data3._evals.length}):`);
       );
     }
   });
-  addRunConfigurationOptions(
-    runs.command("estimate").description("Estimate run cost and duration before starting").argument("<spec-id>", "Behaviour spec ID (full UUID or 8+ char prefix)")
-  ).action(async (specId, cmdOpts) => {
-    const opts = parent.optsWithGlobals();
-    const body = await buildRunRequestBody(cmdOpts, opts);
-    const fullSpecId = await resolveSpecId(specId, opts);
-    const { data: data3 } = await post(
-      `/behavior-specs/${fullSpecId}/runs/estimate`,
-      body,
-      opts
-    );
-    if (isJsonMode()) return printJson(data3);
-    printRunEstimate(data3);
-  });
-  addRunConfigurationOptions(
-    runs.command("start").description("Start a new run for a behaviour spec").argument("<spec-id>", "Behaviour spec ID (full UUID or 8+ char prefix)")
-  ).action(async (specId, cmdOpts) => {
-    const opts = parent.optsWithGlobals();
-    const body = await buildRunRequestBody(cmdOpts, opts);
-    const fullSpecId = await resolveSpecId(specId, opts);
-    const { data: data3 } = await post(
-      `/behavior-specs/${fullSpecId}/runs`,
-      body,
-      opts
-    );
-    if (isJsonMode()) return printJson(data3);
-    printSuccess(
-      `Run #${data3.run_number} started (${shortId(data3.id)}). ${formatProgress(data3) ?? `Status: ${formatStatus(data3.status)}`}`
-    );
-  });
+  for (const name of ["start", "estimate"]) {
+    runs.command(name).description("Retired: use the local pipeline with your own AWS GPU").argument("[spec-id]").action(() => {
+      throw new Error("Hosted training is retired. Configure gpu in local-runner.json and use tt run or tt pipeline run. See docs/local-runtime/aws-gpu.md.");
+    });
+  }
   runs.command("cancel").description("Cancel a running run").argument("<id>", "Run ID (full UUID or 8+ char prefix)").action(async (id) => {
     const opts = parent.optsWithGlobals();
     const fullId = await resolveRunId(id, opts);
@@ -10829,7 +10741,7 @@ async function runCli(version, runtime = {}) {
 }
 
 // src/index.ts
-runCli("0.15.1").catch((err) => {
+runCli("0.16.0").catch((err) => {
   reportError(err);
   process.exit(1);
 });