@tuned-tensor/cli 0.4.21 → 0.4.23

package/README.md

@@ -1,8 +1,11 @@
 # tt - Tuned Tensor CLI
 
-`tt` is the command-line tool for [Tuned Tensor](https://www.tunedtensor.com).
-Use it to define behaviour specs, validate them, launch fine-tuning runs,
-manage datasets, and download or serve trained models.
+`tt` is the command-line client for the optional managed
+[Tuned Tensor](https://www.tunedtensor.com) service. Use it to define behaviour
+specs, launch managed fine-tuning runs, inspect paired baseline-vs-tuned
+reports, and download or serve trained models. For local CUDA or DGX Spark
+training without a Tuned Tensor account, use `tt-local` from
+`@tuned-tensor/local`.
 
 The main CLI documentation lives at
 [tunedtensor.com/docs/cli](https://tunedtensor.com/docs/cli). This README is a
@@ -37,6 +40,7 @@ tt push
 tt runs estimate <spec-id>
 tt runs start <spec-id>
 tt runs watch <run-id>
+tt runs report <run-id>
 ```
 
 To continue training from a completed fine-tuned model artifact:

package/dist/index.js

@@ -625,6 +625,14 @@ function formatEvalScore(run) {
 function formatPercent(rate) {
   return rate == null ? void 0 : (rate * 100).toFixed(1) + "%";
 }
+function formatPointDelta(delta) {
+  if (delta == null) return void 0;
+  const points = delta * 100;
+  return `${points >= 0 ? "+" : ""}${points.toFixed(1)} pp`;
+}
+function formatScore(score) {
+  return score == null ? "\u2014" : score.toFixed(2);
+}
 function formatCountRate(count, total, rate) {
   if (count == null || total == null || rate == null) return void 0;
   return `${formatPercent(rate)} (${count}/${total})`;
@@ -644,6 +652,7 @@ function formatEstimateRange(estimate) {
   return `${formatMinutes(duration.estimated_minutes)} (${formatMinutes(duration.range_minutes.low)}-${formatMinutes(duration.range_minutes.high)})`;
 }
 function printRunEstimate(estimate) {
+  const billing = estimate.billing;
   printDetail([
     ["Estimated Time", formatEstimateRange(estimate)],
     ["Confidence", estimate.duration.confidence],
@@ -651,7 +660,20 @@ function printRunEstimate(estimate) {
     ["Basis", estimate.duration.basis.replaceAll("_", " ")],
     ["Estimated Cost", formatCents(estimate.estimated_cost_cents)],
     ["Training Tokens", `${(estimate.estimated_training_tokens / 1e3).toFixed(1)}k`],
-    ["Epochs", String(estimate.estimated_epochs)]
+    ["Epochs", String(estimate.estimated_epochs)],
+    ["Plan", billing?.plan],
+    [
+      "Billing Source",
+      billing?.billing_source === "free_quota" ? "Free monthly quota" : billing?.billing_source === "credits" ? "Credits" : void 0
+    ],
+    [
+      "Free Runs",
+      billing ? `${billing.free_runs_remaining}/${billing.free_runs_monthly_limit} remaining` : void 0
+    ],
+    [
+      "Free Eligible",
+      billing ? billing.free_run_eligible ? "yes" : `no (${billing.free_run_ineligibility.join(", ") || "not eligible"})` : void 0
+    ]
   ]);
   console.log(
     "\nDuration is a rough historical range. Final cost uses provider-reported training tokens."
@@ -762,6 +784,131 @@ function printEvalOutputDiagnostics(diagnostics) {
     }
   }
 }
+function compactWhitespace(value) {
+  return value.replace(/\s+/g, " ").trim();
+}
+function parseJsonish(value) {
+  if (!value) return value;
+  try {
+    return JSON.parse(value);
+  } catch {
+    return value;
+  }
+}
+function formatJsonish(value, max = 360) {
+  const parsed = parseJsonish(value);
+  const rendered = typeof parsed === "string" ? compactWhitespace(parsed) : JSON.stringify(parsed);
+  return truncate(rendered ?? "\u2014", max);
+}
+function formatPromptSummary(prompt, max = 260) {
+  const subject = prompt.match(/^Subject:\s*(.+)$/m)?.[1]?.trim();
+  const body = prompt.match(/^Body:\s*([\s\S]+)$/m)?.[1];
+  if (subject) {
+    const bodySummary = body ? ` | Body: ${compactWhitespace(body)}` : "";
+    return truncate(`Subject: ${subject}${bodySummary}`, max);
+  }
+  return truncate(compactWhitespace(prompt), max);
+}
+function indexResults(results) {
+  return new Map((results ?? []).map((result) => [result.prompt, result]));
+}
+function printReportMetrics(label, split) {
+  if (!split) return;
+  const comparison = split.comparison;
+  const baseline = split.baseline;
+  const candidate = split.candidate;
+  console.log(`
+${label} Metrics:`);
+  printDetail([
+    ["Base Avg", formatPercent(baseline?.avg_score)],
+    ["Tuned Avg", formatPercent(candidate?.avg_score)],
+    ["Avg Delta", formatPointDelta(comparison?.avg_score_delta)],
+    ["Base Pass", formatPercent(baseline?.pass_rate)],
+    ["Tuned Pass", formatPercent(candidate?.pass_rate)],
+    ["Pass Delta", formatPointDelta(comparison?.pass_rate_delta)],
+    ["Regressions", comparison?.regressions == null ? void 0 : String(comparison.regressions)],
+    ["Improvements", comparison?.improvements == null ? void 0 : String(comparison.improvements)],
+    ["Eval Rows", candidate?.eval_examples_used == null ? candidate?.total == null ? void 0 : String(candidate.total) : String(candidate.eval_examples_used)]
+  ]);
+}
+function printComparedExample(input) {
+  const { index, prompt, baseline, candidate, oldScore, newScore } = input;
+  const expected = candidate?.expected ?? baseline?.expected;
+  const scoreLine = oldScore != null || newScore != null ? oldScore == null ? `tuned score ${formatScore(newScore)}` : `score ${formatScore(oldScore)} -> ${formatScore(newScore)}` : `tuned score ${formatScore(candidate?.score)}`;
+  console.log(`
+${index}. ${scoreLine}`);
+  console.log(`   Prompt: ${formatPromptSummary(prompt)}`);
+  console.log(`   Expected: ${formatJsonish(expected)}`);
+  if (baseline?.actual != null) {
+    console.log(`   Base: ${formatJsonish(baseline.actual)}`);
+  }
+  if (candidate?.actual != null) {
+    console.log(`   Tuned: ${formatJsonish(candidate.actual)}`);
+  }
+  const reasoning = candidate?.reasoning ?? baseline?.reasoning;
+  if (reasoning) {
+    console.log(`   Judge: ${truncate(compactWhitespace(reasoning), 420)}`);
+  }
+}
+function printReportExamples(label, split, mode, limit) {
+  if (!split || limit <= 0) return;
+  const baselineByPrompt = indexResults(split.baseline?.results);
+  const candidateByPrompt = indexResults(split.candidate?.results);
+  if (mode === "regressions") {
+    const regressions = split.comparison?.regressed_examples ?? [];
+    console.log(`
+${label} Regressions:`);
+    if (!regressions.length) {
+      console.log("  No regressed examples in the report.");
+      return;
+    }
+    regressions.slice(0, limit).forEach((example, i) => {
+      printComparedExample({
+        index: i + 1,
+        prompt: example.prompt,
+        baseline: baselineByPrompt.get(example.prompt),
+        candidate: candidateByPrompt.get(example.prompt),
+        oldScore: example.old_score,
+        newScore: example.new_score
+      });
+    });
+    return;
+  }
+  const failures = (split.candidate?.results ?? []).filter((result) => result.passed === false || (result.score ?? 1) < 1).sort((a, b) => (a.score ?? 0) - (b.score ?? 0));
+  console.log(`
+${label} Tuned Failures:`);
+  if (!failures.length) {
+    console.log("  No tuned failures in the report.");
+    return;
+  }
+  failures.slice(0, limit).forEach((candidate, i) => {
+    printComparedExample({
+      index: i + 1,
+      prompt: candidate.prompt,
+      baseline: baselineByPrompt.get(candidate.prompt),
+      candidate,
+      newScore: candidate.score
+    });
+  });
+}
+function printRunReport(report, options) {
+  printDetail([
+    ["Run", report.run_id ? shortId(report.run_id) : void 0],
+    ["Status", report.status ? formatStatus(report.status) : void 0],
+    ["Model", report.fine_tuned_model_id ? shortId(report.fine_tuned_model_id) : void 0]
+  ]);
+  const splits = [];
+  if (options.split === "primary" || options.split === "all") {
+    splits.push(["Primary", report]);
+  }
+  if (options.split === "test" || options.split === "all") {
+    splits.push(["Test", report.test]);
+  }
+  for (const [label, split] of splits) {
+    printReportMetrics(label, split);
+    printReportExamples(label, split, options.mode, options.limit);
+  }
+}
 function addRunConfigurationOptions(command) {
   return command.option("--no-augment", "Disable data augmentation").option("--no-llm-judge", "Disable LLM judging").option("--epochs <n>", "Number of training epochs").option("--lr <rate>", "Learning rate").option("--batch-size <n>", "Batch size").option("--dataset <id>", "Dataset ID to use instead of inline spec examples (full UUID or 4+ char prefix)").option("--parent-model <id>", "Fine-tuned model ID to continue training from (full UUID or 4+ char prefix)").option("--train-ratio <ratio>", "Dataset training split ratio (default: 0.8 when any split ratio is set)").option("--validation-ratio <ratio>", "Dataset validation split ratio (default: 0.1 when any split ratio is set)").option("--test-ratio <ratio>", "Dataset test split ratio (default: 0.1 when any split ratio is set)").option("--lora-rank <n>", "LoRA rank").option("--lora-alpha <n>", "LoRA alpha").option("--max-eval-examples <n>", "Max examples for the primary eval pass").option("--max-test-eval-examples <n>", "Max examples for the secondary test eval pass").option("--long-examples <policy>", "Long training row policy: error, truncate, or skip").option("--max-seq-length <tokens>", "Maximum training sequence length in tokens").option("--max-output-tokens <tokens>", "Desired evaluation output budget in tokens").option("--eval-reserved-output-tokens <tokens>", "Minimum evaluation output tokens reserved per row");
 }
@@ -959,6 +1106,33 @@ Eval Results (${data._evals.length}):`);
     if (isJsonMode()) return printJson(data);
     printDiagnostics(data);
   });
+  runs.command("report").description("Show run metrics and side-by-side eval output insights").argument("<id>", "Run ID (full UUID or 8+ char prefix)").option("--split <split>", "Split to show: primary, test, or all", "primary").option("--limit <n>", "Number of examples to show", "5").option("--mode <mode>", "Example mode: regressions or failures", "regressions").action(async (id, cmdOpts) => {
+    const opts = parent.opts();
+    const fullId = await resolveRunId(id, opts);
+    const { data } = await get(
+      `/runs/${fullId}/report`,
+      void 0,
+      opts
+    );
+    if (isJsonMode()) return printJson(data);
+    const split = String(cmdOpts.split);
+    if (!["primary", "test", "all"].includes(split)) {
+      throw new Error("--split must be one of: primary, test, all");
+    }
+    const mode = String(cmdOpts.mode);
+    if (!["regressions", "failures"].includes(mode)) {
+      throw new Error("--mode must be one of: regressions, failures");
+    }
+    const limit = Number(cmdOpts.limit);
+    if (!Number.isFinite(limit) || limit < 0) {
+      throw new Error("--limit must be a non-negative number");
+    }
+    printRunReport(data, {
+      split,
+      mode,
+      limit: Math.floor(limit)
+    });
+  });
 }
 
 // src/commands/datasets.ts
@@ -3963,7 +4137,7 @@ function registerPushCommand(parent) {
 
 // src/index.ts
 var program = new Command();
-program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.21").option("-k, --api-key <key>", "API key (overrides stored key)").option(
+program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.23").option("-k, --api-key <key>", "API key (overrides stored key)").option(
   "-u, --base-url <url>",
   "API base URL (default: https://tunedtensor.com)"
 ).option("--json", "Output raw JSON").option("--no-color", "Disable colors").hook("preAction", (_thisCommand, actionCommand) => {