@tuned-tensor/cli 0.4.16 → 0.4.17

package/README.md

@@ -55,9 +55,9 @@ tt models base
 tt balance
 ```
 
-For the full command reference, including dataset-backed runs, continued
-fine-tuning, evaluation caps, local model serving, configuration, and billing, see the
-[CLI docs](https://tunedtensor.com/docs/cli).
+For the full command reference, including dataset-backed runs, long-example
+policies, continued fine-tuning, evaluation caps, local model serving,
+configuration, and billing, see the [CLI docs](https://tunedtensor.com/docs/cli).
 
 ## Development
 

package/dist/index.js

@@ -443,7 +443,8 @@ var SPEC_BODY_KEYS = /* @__PURE__ */ new Set([
   "system_prompt",
   "guidelines",
   "constraints",
-  "examples"
+  "examples",
+  "eval_cases"
 ]);
 var RUN_INPUT_KEYS = ["run_id", "behavior_spec_id", "spec_snapshot", "run_number"];
 var SpecBodyError = class extends Error {
@@ -583,6 +584,15 @@ function registerSpecsCommands(parent) {
 
 // src/commands/runs.ts
 import ora from "ora";
+var LONG_EXAMPLE_POLICIES = ["error", "truncate", "skip"];
+function parseLongExamplesPolicy(value) {
+  if (LONG_EXAMPLE_POLICIES.includes(value)) {
+    return value;
+  }
+  throw new Error(
+    `--long-examples must be one of: ${LONG_EXAMPLE_POLICIES.join(", ")}`
+  );
+}
 function formatProgress(run) {
   if (run.progress_pct == null && !run.stage_label) return void 0;
   const label = run.stage_label ?? run.current_stage ?? "Progress";
@@ -791,7 +801,7 @@ Eval Results (${data._evals.length}):`);
       );
     }
   });
-  runs.command("start").description("Start a new run for a behaviour spec").argument("<spec-id>", "Behaviour spec ID (full UUID or 8+ char prefix)").option("--no-augment", "Disable data augmentation").option("--no-llm-judge", "Disable LLM judging").option("--epochs <n>", "Number of training epochs").option("--lr <rate>", "Learning rate").option("--batch-size <n>", "Batch size").option("--dataset <id>", "Dataset ID to use instead of inline spec examples (full UUID or 4+ char prefix)").option("--parent-model <id>", "Fine-tuned model ID to continue training from (full UUID or 4+ char prefix)").option("--train-ratio <ratio>", "Dataset training split ratio (default: 0.8 when any split ratio is set)").option("--validation-ratio <ratio>", "Dataset validation split ratio (default: 0.1 when any split ratio is set)").option("--test-ratio <ratio>", "Dataset test split ratio (default: 0.1 when any split ratio is set)").option("--lora-rank <n>", "LoRA rank").option("--lora-alpha <n>", "LoRA alpha").option("--max-eval-examples <n>", "Max examples for the primary eval pass").option("--max-test-eval-examples <n>", "Max examples for the secondary test eval pass").action(async (specId, cmdOpts) => {
+  runs.command("start").description("Start a new run for a behaviour spec").argument("<spec-id>", "Behaviour spec ID (full UUID or 8+ char prefix)").option("--no-augment", "Disable data augmentation").option("--no-llm-judge", "Disable LLM judging").option("--epochs <n>", "Number of training epochs").option("--lr <rate>", "Learning rate").option("--batch-size <n>", "Batch size").option("--dataset <id>", "Dataset ID to use instead of inline spec examples (full UUID or 4+ char prefix)").option("--parent-model <id>", "Fine-tuned model ID to continue training from (full UUID or 4+ char prefix)").option("--train-ratio <ratio>", "Dataset training split ratio (default: 0.8 when any split ratio is set)").option("--validation-ratio <ratio>", "Dataset validation split ratio (default: 0.1 when any split ratio is set)").option("--test-ratio <ratio>", "Dataset test split ratio (default: 0.1 when any split ratio is set)").option("--lora-rank <n>", "LoRA rank").option("--lora-alpha <n>", "LoRA alpha").option("--max-eval-examples <n>", "Max examples for the primary eval pass").option("--max-test-eval-examples <n>", "Max examples for the secondary test eval pass").option("--long-examples <policy>", "Long training row policy: error, truncate, or skip").option("--max-seq-length <tokens>", "Maximum training sequence length in tokens").action(async (specId, cmdOpts) => {
     const opts = parent.opts();
     const body = {};
     if (cmdOpts.augment === false) body.augment = false;
@@ -819,6 +829,8 @@ Eval Results (${data._evals.length}):`);
     if (cmdOpts.loraAlpha) hp.lora_alpha = Number(cmdOpts.loraAlpha);
     if (cmdOpts.maxEvalExamples) hp.max_eval_examples = Number(cmdOpts.maxEvalExamples);
     if (cmdOpts.maxTestEvalExamples) hp.max_test_eval_examples = Number(cmdOpts.maxTestEvalExamples);
+    if (cmdOpts.longExamples) hp.long_examples = parseLongExamplesPolicy(cmdOpts.longExamples);
+    if (cmdOpts.maxSeqLength) hp.max_seq_length = Number(cmdOpts.maxSeqLength);
     if (cmdOpts.llmJudge === false) body.use_llm_judge = false;
     if (Object.keys(hp).length) body.hyperparameters = hp;
     const fullSpecId = await resolveSpecId(specId, opts);
@@ -2784,11 +2796,90 @@ function validateSpec(spec) {
       message: violations.length === 0 ? void 0 : violations.join("; ")
     });
   }
+  const evalCaseErrors = validateEvalCases(spec);
+  checks.push({
+    name: "Eval cases valid",
+    passed: evalCaseErrors.length === 0,
+    message: evalCaseErrors.length === 0 ? spec.eval_cases?.length ? `${spec.eval_cases.length} executable eval case(s)` : void 0 : evalCaseErrors.join("; ")
+  });
   return {
     valid: checks.every((c) => c.passed),
     checks
   };
 }
+function validateEvalCases(spec) {
+  const errors = [];
+  const cases = spec.eval_cases ?? [];
+  for (const [caseIndex, evalCase] of cases.entries()) {
+    const label = `eval_cases[${caseIndex}]`;
+    if (!evalCase || typeof evalCase !== "object") {
+      errors.push(`${label} must be an object`);
+      continue;
+    }
+    if (typeof evalCase.input !== "string" || evalCase.input.trim().length === 0) {
+      errors.push(`${label}.input must be a non-empty string`);
+    }
+    if (evalCase.runtime !== "python") {
+      errors.push(`${label}.runtime must be "python"`);
+    }
+    if (!Array.isArray(evalCase.tests) || evalCase.tests.length === 0) {
+      errors.push(`${label}.tests must contain at least one test`);
+      continue;
+    }
+    for (const [testIndex, test] of evalCase.tests.entries()) {
+      const testLabel = `${label}.tests[${testIndex}]`;
+      if (!test || typeof test !== "object") {
+        errors.push(`${testLabel} must be an object`);
+        continue;
+      }
+      if (test.name !== void 0 && (typeof test.name !== "string" || test.name.trim().length === 0)) {
+        errors.push(`${testLabel}.name must be a non-empty string when provided`);
+      }
+      if (test.args !== void 0 && (!Array.isArray(test.args) || !test.args.every((arg) => typeof arg === "string"))) {
+        errors.push(`${testLabel}.args must be an array of strings`);
+      }
+      if (test.stdin !== void 0 && typeof test.stdin !== "string") {
+        errors.push(`${testLabel}.stdin must be a string`);
+      }
+      if (test.expected_exit_code !== void 0 && (typeof test.expected_exit_code !== "number" || !Number.isInteger(test.expected_exit_code) || test.expected_exit_code < 0 || test.expected_exit_code > 255)) {
+        errors.push(`${testLabel}.expected_exit_code must be an integer between 0 and 255`);
+      }
+      if (test.expected_stdout !== void 0 && typeof test.expected_stdout !== "string") {
+        errors.push(`${testLabel}.expected_stdout must be a string`);
+      }
+      validateFiles(test.files, `${testLabel}.files`, errors);
+      validateFiles(test.expected_files, `${testLabel}.expected_files`, errors);
+    }
+  }
+  return errors;
+}
+function validateFiles(files, label, errors) {
+  if (files === void 0) return;
+  if (!Array.isArray(files)) {
+    errors.push(`${label} must be an array`);
+    return;
+  }
+  for (const [index, file] of files.entries()) {
+    const fileLabel = `${label}[${index}]`;
+    if (!file || typeof file !== "object") {
+      errors.push(`${fileLabel} must be an object`);
+      continue;
+    }
+    const row = file;
+    if (typeof row.path !== "string" || !isSafeRelativePath(row.path)) {
+      errors.push(`${fileLabel}.path must be a safe relative path`);
+    }
+    if (row.content !== void 0 && typeof row.content !== "string") {
+      errors.push(`${fileLabel}.content must be a string`);
+    }
+  }
+}
+function isSafeRelativePath(value) {
+  if (value.length === 0 || value.length > 240) return false;
+  if (value.includes("\0") || value.startsWith("/")) return false;
+  const normalized = value.replace(/\\/g, "/");
+  return !normalized.split("/").some((part) => part === "" || part === "." || part === "..");
+}
 function checkExamplesAgainstConstraints(spec) {
   const violations = [];
   for (const [i, ex] of spec.examples.entries()) {
@@ -2862,7 +2953,11 @@ function registerPushCommand(parent) {
     const opts = parent.opts();
     const filePath = resolve3(cmdOpts.file);
     const spec = loadSpec(cmdOpts.file);
-    const { id, eval_cases, ...rawBody } = spec;
+    const evalCaseErrors = validateEvalCases(spec);
+    if (evalCaseErrors.length > 0) {
+      throw new Error(`Invalid eval_cases: ${evalCaseErrors.join("; ")}`);
+    }
+    const { id, ...rawBody } = spec;
     const body = canonicalizeSpecBaseModel(rawBody);
     let data;
     if (id) {
@@ -2884,7 +2979,7 @@ function registerPushCommand(parent) {
 
 // src/index.ts
 var program = new Command();
-program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.16").option("-k, --api-key <key>", "API key (overrides stored key)").option(
+program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.17").option("-k, --api-key <key>", "API key (overrides stored key)").option(
   "-u, --base-url <url>",
   "API base URL (default: https://tunedtensor.com)"
 ).option("--json", "Output raw JSON").option("--no-color", "Disable colors").hook("preAction", (_thisCommand, actionCommand) => {