@tuned-tensor/cli 0.4.18 → 0.4.19

package/README.md

@@ -34,6 +34,7 @@ tt init --name "Customer Support Bot" --model Qwen/Qwen3.5-2B
 # Edit tunedtensor.json, then:
 tt eval
 tt push
+tt runs estimate <spec-id>
 tt runs start <spec-id>
 tt runs watch <run-id>
 ```
@@ -72,8 +73,9 @@ This wraps llama.cpp's `convert_hf_to_gguf.py` + `llama-quantize` and Ollama's
 system prompt is embedded as the Modelfile `SYSTEM` block.
 
 For the full command reference, including dataset-backed runs, long-example
-policies, eval token budgets, continued fine-tuning, evaluation caps, local model serving,
-configuration, and billing, see the [CLI docs](https://tunedtensor.com/docs/cli).
+policies, eval token budgets, preflight run estimates, continued fine-tuning,
+evaluation caps, local model serving, configuration, and billing, see the
+[CLI docs](https://tunedtensor.com/docs/cli).
 
 ## Development
 

package/dist/index.js

@@ -617,6 +617,29 @@ function formatMinutes(minutes) {
   if (minutes < 60) return `${Math.max(1, Math.round(minutes))}m`;
   return `${(minutes / 60).toFixed(1)}h`;
 }
+function formatCents(cents) {
+  const dollars = cents / 100;
+  if (Math.abs(dollars) >= 100) return `$${dollars.toFixed(0)}`;
+  return `$${dollars.toFixed(2)}`;
+}
+function formatEstimateRange(estimate) {
+  const duration = estimate.duration;
+  return `${formatMinutes(duration.estimated_minutes)} (${formatMinutes(duration.range_minutes.low)}-${formatMinutes(duration.range_minutes.high)})`;
+}
+function printRunEstimate(estimate) {
+  printDetail([
+    ["Estimated Time", formatEstimateRange(estimate)],
+    ["Confidence", estimate.duration.confidence],
+    ["History Samples", String(estimate.duration.sample_count)],
+    ["Basis", estimate.duration.basis.replaceAll("_", " ")],
+    ["Estimated Cost", formatCents(estimate.estimated_cost_cents)],
+    ["Training Tokens", `${(estimate.estimated_training_tokens / 1e3).toFixed(1)}k`],
+    ["Epochs", String(estimate.estimated_epochs)]
+  ]);
+  console.log(
+    "\nDuration is a rough historical range. Final cost uses provider-reported training tokens."
+  );
+}
 function formatEpochProgress(diagnostics) {
   const curve = diagnostics.training.curve;
   if (curve.latest_epoch == null) return void 0;
@@ -722,6 +745,46 @@ function printEvalOutputDiagnostics(diagnostics) {
     }
   }
 }
+function addRunConfigurationOptions(command) {
+  return command.option("--no-augment", "Disable data augmentation").option("--no-llm-judge", "Disable LLM judging").option("--epochs <n>", "Number of training epochs").option("--lr <rate>", "Learning rate").option("--batch-size <n>", "Batch size").option("--dataset <id>", "Dataset ID to use instead of inline spec examples (full UUID or 4+ char prefix)").option("--parent-model <id>", "Fine-tuned model ID to continue training from (full UUID or 4+ char prefix)").option("--train-ratio <ratio>", "Dataset training split ratio (default: 0.8 when any split ratio is set)").option("--validation-ratio <ratio>", "Dataset validation split ratio (default: 0.1 when any split ratio is set)").option("--test-ratio <ratio>", "Dataset test split ratio (default: 0.1 when any split ratio is set)").option("--lora-rank <n>", "LoRA rank").option("--lora-alpha <n>", "LoRA alpha").option("--max-eval-examples <n>", "Max examples for the primary eval pass").option("--max-test-eval-examples <n>", "Max examples for the secondary test eval pass").option("--long-examples <policy>", "Long training row policy: error, truncate, or skip").option("--max-seq-length <tokens>", "Maximum training sequence length in tokens").option("--max-output-tokens <tokens>", "Desired evaluation output budget in tokens").option("--eval-reserved-output-tokens <tokens>", "Minimum evaluation output tokens reserved per row");
+}
+async function buildRunRequestBody(cmdOpts, opts) {
+  const body = {};
+  if (cmdOpts.augment === false) body.augment = false;
+  if (cmdOpts.dataset) body.dataset_id = await resolveDatasetId(String(cmdOpts.dataset), opts);
+  if (cmdOpts.parentModel) {
+    body.parent_model_id = await resolveModelId(String(cmdOpts.parentModel), opts);
+  }
+  const splitRatioOptions = [
+    cmdOpts.trainRatio,
+    cmdOpts.validationRatio,
+    cmdOpts.testRatio
+  ];
+  if (splitRatioOptions.some((value) => value != null)) {
+    body.split_ratios = {
+      train: cmdOpts.trainRatio != null ? Number(cmdOpts.trainRatio) : 0.8,
+      validation: cmdOpts.validationRatio != null ? Number(cmdOpts.validationRatio) : 0.1,
+      test: cmdOpts.testRatio != null ? Number(cmdOpts.testRatio) : 0.1
+    };
+  }
+  const hp = {};
+  if (cmdOpts.epochs) hp.n_epochs = Number(cmdOpts.epochs);
+  if (cmdOpts.lr) hp.learning_rate = Number(cmdOpts.lr);
+  if (cmdOpts.batchSize) hp.batch_size = Number(cmdOpts.batchSize);
+  if (cmdOpts.loraRank) hp.lora_rank = Number(cmdOpts.loraRank);
+  if (cmdOpts.loraAlpha) hp.lora_alpha = Number(cmdOpts.loraAlpha);
+  if (cmdOpts.maxEvalExamples) hp.max_eval_examples = Number(cmdOpts.maxEvalExamples);
+  if (cmdOpts.maxTestEvalExamples) hp.max_test_eval_examples = Number(cmdOpts.maxTestEvalExamples);
+  if (cmdOpts.longExamples) hp.long_examples = parseLongExamplesPolicy(String(cmdOpts.longExamples));
+  if (cmdOpts.maxSeqLength) hp.max_seq_length = Number(cmdOpts.maxSeqLength);
+  if (cmdOpts.maxOutputTokens) hp.max_output_tokens = Number(cmdOpts.maxOutputTokens);
+  if (cmdOpts.evalReservedOutputTokens) {
+    hp.eval_reserved_output_tokens = Number(cmdOpts.evalReservedOutputTokens);
+  }
+  if (cmdOpts.llmJudge === false) body.use_llm_judge = false;
+  if (Object.keys(hp).length) body.hyperparameters = hp;
+  return body;
+}
 function registerRunsCommands(parent) {
   const runs = parent.command("runs").description("Manage runs");
   runs.command("list").description("List runs").option("-s, --spec <id>", "Filter by spec ID (full UUID or 8+ char prefix)").option("-p, --page <n>", "Page number", "1").option("--per-page <n>", "Results per page", "20").action(async (cmdOpts) => {
@@ -801,42 +864,25 @@ Eval Results (${data._evals.length}):`);
       );
     }
   });
-  runs.command("start").description("Start a new run for a behaviour spec").argument("<spec-id>", "Behaviour spec ID (full UUID or 8+ char prefix)").option("--no-augment", "Disable data augmentation").option("--no-llm-judge", "Disable LLM judging").option("--epochs <n>", "Number of training epochs").option("--lr <rate>", "Learning rate").option("--batch-size <n>", "Batch size").option("--dataset <id>", "Dataset ID to use instead of inline spec examples (full UUID or 4+ char prefix)").option("--parent-model <id>", "Fine-tuned model ID to continue training from (full UUID or 4+ char prefix)").option("--train-ratio <ratio>", "Dataset training split ratio (default: 0.8 when any split ratio is set)").option("--validation-ratio <ratio>", "Dataset validation split ratio (default: 0.1 when any split ratio is set)").option("--test-ratio <ratio>", "Dataset test split ratio (default: 0.1 when any split ratio is set)").option("--lora-rank <n>", "LoRA rank").option("--lora-alpha <n>", "LoRA alpha").option("--max-eval-examples <n>", "Max examples for the primary eval pass").option("--max-test-eval-examples <n>", "Max examples for the secondary test eval pass").option("--long-examples <policy>", "Long training row policy: error, truncate, or skip").option("--max-seq-length <tokens>", "Maximum training sequence length in tokens").option("--max-output-tokens <tokens>", "Desired evaluation output budget in tokens").option("--eval-reserved-output-tokens <tokens>", "Minimum evaluation output tokens reserved per row").action(async (specId, cmdOpts) => {
+  addRunConfigurationOptions(
+    runs.command("estimate").description("Estimate run cost and duration before starting").argument("<spec-id>", "Behaviour spec ID (full UUID or 8+ char prefix)")
+  ).action(async (specId, cmdOpts) => {
     const opts = parent.opts();
-    const body = {};
-    if (cmdOpts.augment === false) body.augment = false;
-    if (cmdOpts.dataset) body.dataset_id = await resolveDatasetId(cmdOpts.dataset, opts);
-    if (cmdOpts.parentModel) {
-      body.parent_model_id = await resolveModelId(cmdOpts.parentModel, opts);
-    }
-    const splitRatioOptions = [
-      cmdOpts.trainRatio,
-      cmdOpts.validationRatio,
-      cmdOpts.testRatio
-    ];
-    if (splitRatioOptions.some((value) => value != null)) {
-      body.split_ratios = {
-        train: cmdOpts.trainRatio != null ? Number(cmdOpts.trainRatio) : 0.8,
-        validation: cmdOpts.validationRatio != null ? Number(cmdOpts.validationRatio) : 0.1,
-        test: cmdOpts.testRatio != null ? Number(cmdOpts.testRatio) : 0.1
-      };
-    }
-    const hp = {};
-    if (cmdOpts.epochs) hp.n_epochs = Number(cmdOpts.epochs);
-    if (cmdOpts.lr) hp.learning_rate = Number(cmdOpts.lr);
-    if (cmdOpts.batchSize) hp.batch_size = Number(cmdOpts.batchSize);
-    if (cmdOpts.loraRank) hp.lora_rank = Number(cmdOpts.loraRank);
-    if (cmdOpts.loraAlpha) hp.lora_alpha = Number(cmdOpts.loraAlpha);
-    if (cmdOpts.maxEvalExamples) hp.max_eval_examples = Number(cmdOpts.maxEvalExamples);
-    if (cmdOpts.maxTestEvalExamples) hp.max_test_eval_examples = Number(cmdOpts.maxTestEvalExamples);
-    if (cmdOpts.longExamples) hp.long_examples = parseLongExamplesPolicy(cmdOpts.longExamples);
-    if (cmdOpts.maxSeqLength) hp.max_seq_length = Number(cmdOpts.maxSeqLength);
-    if (cmdOpts.maxOutputTokens) hp.max_output_tokens = Number(cmdOpts.maxOutputTokens);
-    if (cmdOpts.evalReservedOutputTokens) {
-      hp.eval_reserved_output_tokens = Number(cmdOpts.evalReservedOutputTokens);
-    }
-    if (cmdOpts.llmJudge === false) body.use_llm_judge = false;
-    if (Object.keys(hp).length) body.hyperparameters = hp;
+    const body = await buildRunRequestBody(cmdOpts, opts);
+    const fullSpecId = await resolveSpecId(specId, opts);
+    const { data } = await post(
+      `/behavior-specs/${fullSpecId}/runs/estimate`,
+      body,
+      opts
+    );
+    if (isJsonMode()) return printJson(data);
+    printRunEstimate(data);
+  });
+  addRunConfigurationOptions(
+    runs.command("start").description("Start a new run for a behaviour spec").argument("<spec-id>", "Behaviour spec ID (full UUID or 8+ char prefix)")
+  ).action(async (specId, cmdOpts) => {
+    const opts = parent.opts();
+    const body = await buildRunRequestBody(cmdOpts, opts);
     const fullSpecId = await resolveSpecId(specId, opts);
     const { data } = await post(
       `/behavior-specs/${fullSpecId}/runs`,
@@ -2927,15 +2973,15 @@ var KIND_LABELS = {
   refund: "Refund",
   adjustment: "Adjustment"
 };
-function formatCents(cents) {
+function formatCents2(cents) {
   const sign = cents < 0 ? "-" : "";
   const abs = Math.abs(cents);
   return `${sign}$${(abs / 100).toFixed(2)}`;
 }
 function formatSigned(cents) {
-  if (cents > 0) return chalk3.green(`+${formatCents(cents)}`);
-  if (cents < 0) return chalk3.red(formatCents(cents));
-  return formatCents(cents);
+  if (cents > 0) return chalk3.green(`+${formatCents2(cents)}`);
+  if (cents < 0) return chalk3.red(formatCents2(cents));
+  return formatCents2(cents);
 }
 function registerBalanceCommands(parent) {
   parent.command("balance").description("Show credit balance and recent transactions").option("-n, --limit <n>", "Number of transactions to show (default 10)", "10").action(async (options) => {
@@ -2949,10 +2995,10 @@ function registerBalanceCommands(parent) {
       return printJson({ balance, transactions });
     }
     const lowBalance = balance.balance_cents < 100;
-    const balanceLine = lowBalance ? chalk3.red(formatCents(balance.balance_cents)) + chalk3.dim(" (low)") : chalk3.bold.green(formatCents(balance.balance_cents));
+    const balanceLine = lowBalance ? chalk3.red(formatCents2(balance.balance_cents)) + chalk3.dim(" (low)") : chalk3.bold.green(formatCents2(balance.balance_cents));
     printDetail([
       ["Credits", balanceLine],
-      ["Lifetime top-ups", formatCents(balance.lifetime_topup_cents)]
+      ["Lifetime top-ups", formatCents2(balance.lifetime_topup_cents)]
     ]);
     if (transactions && transactions.length > 0) {
       console.log(`
@@ -2963,7 +3009,7 @@ ${chalk3.bold("Recent transactions")}`);
           formatDate(row.created_at),
           KIND_LABELS[row.kind] ?? row.kind,
           formatSigned(row.amount_cents),
-          formatCents(row.balance_after_cents),
+          formatCents2(row.balance_after_cents),
           row.reference_id ? shortId(row.reference_id) : chalk3.dim("\u2014")
         ])
       );
@@ -3294,7 +3340,7 @@ function registerPushCommand(parent) {
 
 // src/index.ts
 var program = new Command();
-program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.18").option("-k, --api-key <key>", "API key (overrides stored key)").option(
+program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.19").option("-k, --api-key <key>", "API key (overrides stored key)").option(
   "-u, --base-url <url>",
   "API base URL (default: https://tunedtensor.com)"
 ).option("--json", "Output raw JSON").option("--no-color", "Disable colors").hook("preAction", (_thisCommand, actionCommand) => {