@tuned-tensor/cli 0.25.0 → 0.26.1

package/dist/local-runtime.js

@@ -21,6 +21,7 @@ import {
   exampleToChatRow,
   examplesFromChatJsonl,
   examplesFromSpec,
+  executeDatasetSplit,
   fingerprintLocalBaseModel,
   foundationPlaceholderIssues,
   generatedPlaceholderIssues,
@@ -28,6 +29,7 @@ import {
   huggingFaceDataset,
   initLocalRunnerConfigFile,
   initLocalSpecFile,
+  inspectDatasetFile,
   isTerminalRunState,
   loadJsonFile,
   loadLocalRunInput,
@@ -40,6 +42,7 @@ import {
   parseLocalRunnerConfig,
   pipelineForRunInput,
   prefetchHuggingFaceDataset,
+  prepareDatasetSplit,
   resolveDownloadConfig,
   resolveLocalRunInputPaths,
   resolveProjectConfig,
@@ -53,7 +56,7 @@ import {
   validateLocalFineTuneInput,
   validateLocalPipeline,
   writePrivateJsonAtomic
-} from "./chunk-7RJGLRMO.js";
+} from "./chunk-FLRRDP3M.js";
 import {
   applySpecLock,
   assertArtifactManifest,
@@ -122,7 +125,7 @@ import {
   withOfflineHuggingFaceCacheEnvironment,
   withProgressHeartbeat,
   writeSpecLockEntry
-} from "./chunk-UCJCWJ6F.js";
+} from "./chunk-4BHDPRYD.js";
 
 // src/local-runtime/index.ts
 import { readFileSync as readFileSync3, realpathSync } from "fs";
@@ -962,6 +965,7 @@ Commands:
   serve <model-id|active|base|foundation> [--config local-runner.json] [--host 127.0.0.1] [--port 8000]
   runs list|get|events|report|compare|audit [args] [--config local-runner.json]
   models list|get|verify|prefetch|verify-base|active|activate|rollback|serve [args] [--config local-runner.json]
+  datasets inspect|split <data.jsonl> [--output directory --train .8 --validation .1 --test .1]
   datasets prefetch|verify [tunedtensor.json] [--repo org/name --training path ...]
   assets list|path|verify [org/name[@revision]]
 
@@ -1131,9 +1135,33 @@ var COMMAND_GROUPS = {
     }
   },
   datasets: {
-    description: "Download or verify the Hugging Face dataset a behavior spec trains on.",
+    description: "Inspect and split local chat JSONL, or download and verify Hugging Face datasets.",
     defaultSubcommand: "prefetch",
     subcommands: {
+      inspect: {
+        usage: "tt datasets inspect <data.jsonl>",
+        description: "Validate local chat JSONL and report row count, hash, and bounded samples without changes.",
+        options: [],
+        minPositionals: 1,
+        maxPositionals: 1
+      },
+      split: {
+        usage: "tt datasets split <data.jsonl> --output <new-directory> [options]",
+        description: "Partition unsplit chat JSONL without changing the source or spec. Accepts a workspace path or pinned hf://datasets/org/repo@commit/file reference from a recorded download. Keep duplicate prompts together; preserve metadata.",
+        options: [
+          { name: "--output", value: "directory", description: "New output directory inside the workspace (parent must exist)" },
+          { name: "--train", value: "ratio", description: "Training fraction (default 0.8)" },
+          { name: "--validation", value: "ratio", description: "Validation fraction (default 0.1)" },
+          { name: "--test", value: "ratio", description: "Test fraction (default 0.1); set 0 for train/validation only" },
+          { name: "--seed", value: "seed", description: "Reproducible random group assignment (default 42)" },
+          { name: "--method", value: "random|ordered", description: "Seeded group ordering or source order (default random)" },
+          { name: "--group-by", value: "field", description: "Group field, with dotted paths supported; defaults to normalized user prompt" },
+          { name: "--stratify-by", value: "field", description: "Keep similar ratios within each label/category" },
+          { name: "--dry-run", description: "Show planned counts without writing files" }
+        ],
+        minPositionals: 1,
+        maxPositionals: 1
+      },
       prefetch: {
         usage: "tt datasets prefetch [tunedtensor.json] [--repo org/name --training path [--test path]] [options]",
         description: "Download dataset_prebuilt.huggingface files and pin them in tunedtensor.lock.json, or download any dataset files with --repo. Resumes after interruption.",
@@ -2026,6 +2054,22 @@ ${validation.errors.join("\n")}`);
   }
   if (command === "datasets") {
     const subcommand = cli.subcommand;
+    if (subcommand === "inspect") return printJson(await inspectDatasetFile(cwd(), cli.positionals[0]));
+    if (subcommand === "split") {
+      const numeric = (name, fallback) => readOption(argv, name) === void 0 ? fallback : Number(readOption(argv, name));
+      const prepared = await prepareDatasetSplit(cwd(), {
+        file: cli.positionals[0],
+        output: readOption(argv, "--output"),
+        train: numeric("--train", 0.8),
+        validation: numeric("--validation", 0.1),
+        test: numeric("--test", 0.1),
+        seed: readOption(argv, "--seed") ?? "42",
+        method: readOption(argv, "--method") ?? "random",
+        group_by: readOption(argv, "--group-by"),
+        stratify_by: readOption(argv, "--stratify-by")
+      });
+      return printJson(hasFlag(argv, "--dry-run") ? prepared.preview : await executeDatasetSplit(cwd(), prepared));
+    }
     const inputPath = resolve3(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH);
     const config = readOption(argv, "--repo") ? await resolveDownloadConfig(inputPath, readOption(argv, "--config"), (message) => console.error(message)) : await configFromArgv(argv, inputPath);
     if (subcommand !== "prefetch" && subcommand !== "verify") throw new Error(`Unknown datasets command: ${subcommand}`);