@tuned-tensor/cli 0.24.0 → 0.26.0

package/dist/local-runtime.js

@@ -2,113 +2,130 @@
 import {
   DEFAULT_LOCAL_SPEC_PATH,
   DatasetNotDownloadedError,
-  assertArtifactManifest,
   assertFoundationSpecReady,
   assertLocalRunInputReady,
-  assertUsableModelArtifact,
   assessHardware,
   auditRun,
+  buildSystemMessage,
+  canonicalLocalPipeline,
+  checkAwsGpu,
+  compileSpecToJsonl,
+  createLocalStore,
+  createRunConsoleReporter,
+  datasetInputIdentity,
+  decisionPlaceholderIssues,
+  defaultLocalHome,
+  ensureDatasetCached,
+  evaluateGeneralRegressionGate,
+  evaluationSuiteFromChatJsonl,
+  exampleToChatRow,
+  examplesFromChatJsonl,
+  examplesFromSpec,
+  executeDatasetSplit,
+  fingerprintLocalBaseModel,
+  foundationPlaceholderIssues,
+  generatedPlaceholderIssues,
+  hardenExistingLocalStore,
+  huggingFaceDataset,
+  initLocalRunnerConfigFile,
+  initLocalSpecFile,
+  inspectDatasetFile,
+  isTerminalRunState,
+  loadJsonFile,
+  loadLocalRunInput,
+  loadLocalRunnerConfig,
+  loadRunRequest,
+  localPathsOverlap,
+  localStorePaths,
+  normalizeChatJsonlForRelocation,
+  parseLocalRunInput,
+  parseLocalRunnerConfig,
+  pipelineForRunInput,
+  prefetchHuggingFaceDataset,
+  prepareDatasetSplit,
+  resolveDownloadConfig,
+  resolveLocalRunInputPaths,
+  resolveProjectConfig,
+  runGpuProcess,
+  runLocalFineTune,
+  runLocalPipeline,
+  runRequestFromLocalSpec,
+  specHash,
+  tokenF1,
+  validateBehaviorSpec,
+  validateLocalFineTuneInput,
+  validateLocalPipeline,
+  writePrivateJsonAtomic
+} from "./chunk-FLRRDP3M.js";
+import {
+  applySpecLock,
+  assertArtifactManifest,
+  assertUsableModelArtifact,
   baseModelRevisionSchema,
   behaviorSpecExampleSchema,
   behaviorSpecSchema,
   buildBundledPythonCommand,
   buildFoundationPythonCommand,
   buildModelPrefetchPayload,
-  buildSystemMessage,
-  canonicalLocalPipeline,
-  checkAwsGpu,
+  buildModelSourcePrefetchPayload,
   comparisonReportSchema,
-  compileSpecToJsonl,
-  createLocalStore,
-  createRunConsoleReporter,
   datasetColumnsSchema,
   datasetFormatSchema,
-  datasetInputIdentity,
   datasetPrebuiltSchema,
   datasetSourceSchema,
   decisionHyperparametersSchema,
   decisionLabels,
-  decisionPlaceholderIssues,
   decisionQuestionSchema,
   defaultBaseModelRevision,
-  defaultLocalHome,
-  ensureDatasetCached,
+  enforceSpecLock,
   evalExampleResultSchema,
   evalReportSchema,
   evalSplitSchema,
-  evaluateGeneralRegressionGate,
-  evaluationSuiteFromChatJsonl,
-  exampleToChatRow,
-  examplesFromChatJsonl,
-  examplesFromSpec,
+  findAsset,
   fineTuneHyperparametersSchema,
   fineTuneRunRequestSchema,
-  fingerprintLocalBaseModel,
   forwardStreamLines,
   foundationHyperparametersSchema,
-  foundationPlaceholderIssues,
-  generatedPlaceholderIssues,
-  hardenExistingLocalStore,
-  huggingFaceDataset,
   huggingFaceDatasetSourceSchema,
   huggingFaceRepoIdSchema,
-  initLocalRunnerConfigFile,
-  initLocalSpecFile,
   isDecisionSpecFile,
   isFoundationSpecFile,
   isRepoRelativePath,
-  isTerminalRunState,
   jsonFieldMetricsSchema,
-  loadJsonFile,
-  loadLocalRunInput,
-  loadLocalRunnerConfig,
-  loadRunRequest,
+  listAssets,
   localAdapterSpecFileSchema,
   localBehaviorSpecFileSchema,
   localDecisionSpecFileSchema,
   localFoundationSpecFileSchema,
-  localPathsOverlap,
   localRunnerConfigSchema,
   localRuntimePackageRoot,
-  localStorePaths,
   minimalMachineLearningEnvironment,
-  normalizeChatJsonlForRelocation,
-  parseLocalRunInput,
-  parseLocalRunnerConfig,
-  pipelineForRunInput,
   prefetchBaseModel,
-  prefetchHuggingFaceDataset,
   projectRuntimeSchema,
   pythonEnvironmentPath,
+  readSpecLock,
   reportInBackground,
-  resolveLocalRunInputPaths,
   resolveModelLoader,
-  resolveProjectConfig,
   resolveRequestedBaseModelRevision,
   resolveTrainingModel,
-  runGpuProcess,
-  runLocalFineTune,
-  runLocalPipeline,
   runProvenanceSchema,
   runReportSchema,
-  runRequestFromLocalSpec,
   safeName,
   sanitizeLogLine,
-  specHash,
+  snapshotDigests,
+  specLockPath,
   specSnapshotSchema,
-  tokenF1,
   trainingReportSchema,
-  validateBehaviorSpec,
-  validateLocalFineTuneInput,
-  validateLocalPipeline,
+  verifyAsset,
   verifyLocalBaseModel,
   verifyModelPrefetchCacheReport,
   withBundledPythonEnvironment,
   withFoundationPythonEnvironment,
   withHuggingFaceCacheEnvironment,
   withOfflineHuggingFaceCacheEnvironment,
-  writePrivateJsonAtomic
-} from "./chunk-RPWOCVIL.js";
+  withProgressHeartbeat,
+  writeSpecLockEntry
+} from "./chunk-4BHDPRYD.js";
 
 // src/local-runtime/index.ts
 import { readFileSync as readFileSync3, realpathSync } from "fs";
@@ -948,7 +965,9 @@ Commands:
   serve <model-id|active|base|foundation> [--config local-runner.json] [--host 127.0.0.1] [--port 8000]
   runs list|get|events|report|compare|audit [args] [--config local-runner.json]
   models list|get|verify|prefetch|verify-base|active|activate|rollback|serve [args] [--config local-runner.json]
-  datasets prefetch|verify [tunedtensor.json]
+  datasets inspect|split <data.jsonl> [--output directory --train .8 --validation .1 --test .1]
+  datasets prefetch|verify [tunedtensor.json] [--repo org/name --training path ...]
+  assets list|path|verify [org/name[@revision]]
 
 Global options:
   -h, --help                       Show help
@@ -961,6 +980,9 @@ TT_LOCAL_HOME overrides the default store. --config is a legacy migration option
 var CONFIG_OPTION = { name: "--config", value: "path", description: "Legacy runner config (prefer runtime/evaluation in tunedtensor.json)" };
 var VERBOSE_OPTION = { name: "--verbose", description: "Stream subprocess output" };
 var QUIET_OPTION = { name: "--quiet", description: "Suppress progress output on stderr" };
+var REPO_OPTION = { name: "--repo", value: "org/name", description: "Download this Hugging Face repo instead of the spec's (records a manifest; leaves the lock alone)" };
+var REVISION_OPTION = { name: "--revision", value: "sha", description: "40-character commit to download with --repo (default: latest)" };
+var SPLIT_OPTIONS = ["training", "validation", "test"].map((split) => ({ name: `--${split}`, value: "path", description: `Repo-relative ${split} file (.jsonl or .parquet) to download with --repo` }));
 var MODEL_SERVE_OPTIONS = [
   CONFIG_OPTION,
   { name: "--checkpoint", value: "path", description: "Foundation model directory (target: foundation)" },
@@ -1071,9 +1093,9 @@ var COMMAND_GROUPS = {
         missingPositionalsMessage: "models verify requires <model-id-or-artifact-path>"
       },
       prefetch: {
-        usage: "tt models prefetch [tunedtensor.json] [options]",
-        description: "Download the configured base model before a run.",
-        options: [CONFIG_OPTION, VERBOSE_OPTION, QUIET_OPTION],
+        usage: "tt models prefetch [tunedtensor.json] [--repo org/name [--revision sha]] [options]",
+        description: "Download the spec's base model and pin it in tunedtensor.lock.json, or download any model with --repo.",
+        options: [CONFIG_OPTION, REPO_OPTION, REVISION_OPTION, VERBOSE_OPTION, QUIET_OPTION],
         maxPositionals: 1
       },
       "verify-base": {
@@ -1113,13 +1135,37 @@ var COMMAND_GROUPS = {
     }
   },
   datasets: {
-    description: "Download or verify the Hugging Face dataset a behavior spec trains on.",
+    description: "Inspect and split local chat JSONL, or download and verify Hugging Face datasets.",
     defaultSubcommand: "prefetch",
     subcommands: {
+      inspect: {
+        usage: "tt datasets inspect <data.jsonl>",
+        description: "Validate local chat JSONL and report row count, hash, and bounded samples without changes.",
+        options: [],
+        minPositionals: 1,
+        maxPositionals: 1
+      },
+      split: {
+        usage: "tt datasets split <data.jsonl> --output <new-directory> [options]",
+        description: "Partition unsplit chat JSONL without changing the source or spec. Accepts a workspace path or pinned hf://datasets/org/repo@commit/file reference from a recorded download. Keep duplicate prompts together; preserve metadata.",
+        options: [
+          { name: "--output", value: "directory", description: "New output directory inside the workspace (parent must exist)" },
+          { name: "--train", value: "ratio", description: "Training fraction (default 0.8)" },
+          { name: "--validation", value: "ratio", description: "Validation fraction (default 0.1)" },
+          { name: "--test", value: "ratio", description: "Test fraction (default 0.1); set 0 for train/validation only" },
+          { name: "--seed", value: "seed", description: "Reproducible random group assignment (default 42)" },
+          { name: "--method", value: "random|ordered", description: "Seeded group ordering or source order (default random)" },
+          { name: "--group-by", value: "field", description: "Group field, with dotted paths supported; defaults to normalized user prompt" },
+          { name: "--stratify-by", value: "field", description: "Keep similar ratios within each label/category" },
+          { name: "--dry-run", description: "Show planned counts without writing files" }
+        ],
+        minPositionals: 1,
+        maxPositionals: 1
+      },
       prefetch: {
-        usage: "tt datasets prefetch [tunedtensor.json] [options]",
-        description: "Download dataset_prebuilt.huggingface files into the local Hugging Face cache. Resumes after interruption.",
-        options: [CONFIG_OPTION, VERBOSE_OPTION, QUIET_OPTION],
+        usage: "tt datasets prefetch [tunedtensor.json] [--repo org/name --training path [--test path]] [options]",
+        description: "Download dataset_prebuilt.huggingface files and pin them in tunedtensor.lock.json, or download any dataset files with --repo. Resumes after interruption.",
+        options: [CONFIG_OPTION, REPO_OPTION, REVISION_OPTION, ...SPLIT_OPTIONS, VERBOSE_OPTION, QUIET_OPTION],
         maxPositionals: 1
       },
       verify: {
@@ -1129,6 +1175,27 @@ var COMMAND_GROUPS = {
         maxPositionals: 1
       }
     }
+  },
+  assets: {
+    description: "Find and verify downloaded Hugging Face datasets and models (manifests under ~/.tuned-tensor/assets).",
+    defaultSubcommand: "list",
+    subcommands: {
+      list: { usage: "tt assets list", description: "List downloads, newest first.", options: [], maxPositionals: 0 },
+      path: {
+        usage: "tt assets path <org/name[@revision]>",
+        description: "Print where a download lives on disk.",
+        options: [],
+        minPositionals: 1,
+        maxPositionals: 1,
+        missingPositionalsMessage: "assets path requires <org/name[@revision]>"
+      },
+      verify: {
+        usage: "tt assets verify [org/name[@revision]]",
+        description: "Re-hash downloaded files and compare them with their manifests (all downloads by default).",
+        options: [],
+        maxPositionals: 1
+      }
+    }
   }
 };
 function hasHelpFlag(argv) {
@@ -1254,6 +1321,26 @@ function parseCli(argv) {
     definition: subcommandDefinition
   };
 }
+function datasetSourceFromArgv(argv) {
+  const repo = readOption(argv, "--repo");
+  if (!repo) return void 0;
+  const revision = readOption(argv, "--revision");
+  const splits = Object.fromEntries(["training", "validation", "test"].flatMap((split) => {
+    const path = readOption(argv, `--${split}`);
+    return path ? [[split, path]] : [];
+  }));
+  if (!splits.training) {
+    throw new Error("--repo needs --training <path> (a repo-relative .jsonl or .parquet file); --validation and --test are optional.");
+  }
+  const invalid = Object.values(splits).find((path) => !isRepoRelativePath(path));
+  if (invalid) throw new Error(`Dataset files must be paths inside the repo, such as data/train.jsonl: ${invalid}`);
+  return {
+    format: "chat_jsonl",
+    huggingface: huggingFaceDatasetSourceSchema.parse({ repo, ...revision ? { revision } : {} }),
+    ...splits,
+    training: splits.training
+  };
+}
 function readNumberOption(argv, name) {
   const value = readOption(argv, name);
   return value ? Number(value) : void 0;
@@ -1866,7 +1953,8 @@ ${validation.errors.join("\n")}`);
       verbose: hasFlag(argv, "--verbose"),
       quiet: hasFlag(argv, "--quiet")
     });
-    const ensured = await ensureDatasetCached({ request: input.request, config, reporter });
+    const lock = await readSpecLock(input.path);
+    const ensured = await ensureDatasetCached({ request: applySpecLock(input.request, lock), config, reporter });
     const validated = await validateLocalFineTuneInput({
       request: ensured.request,
       config
@@ -1889,6 +1977,7 @@ ${validation.errors.join("\n")}`);
         }
       });
     }
+    if (!config.dryRun) await enforceSpecLock({ specPath: input.path, request, config, reporter });
     const specFile = /^\..+\.tt-local-/.test(basename(input.path)) ? void 0 : { path: input.path, sha256: specHash(await readFile2(input.path, "utf8")) };
     if (input.spec.pipeline) {
       const result2 = await runLocalPipeline({ request, config, reporter, pipeline: pipelineForRunInput(input), projectSpec: input.spec, specFile });
@@ -1965,25 +2054,74 @@ ${validation.errors.join("\n")}`);
   }
   if (command === "datasets") {
     const subcommand = cli.subcommand;
+    if (subcommand === "inspect") return printJson(await inspectDatasetFile(cwd(), cli.positionals[0]));
+    if (subcommand === "split") {
+      const numeric = (name, fallback) => readOption(argv, name) === void 0 ? fallback : Number(readOption(argv, name));
+      const prepared = await prepareDatasetSplit(cwd(), {
+        file: cli.positionals[0],
+        output: readOption(argv, "--output"),
+        train: numeric("--train", 0.8),
+        validation: numeric("--validation", 0.1),
+        test: numeric("--test", 0.1),
+        seed: readOption(argv, "--seed") ?? "42",
+        method: readOption(argv, "--method") ?? "random",
+        group_by: readOption(argv, "--group-by"),
+        stratify_by: readOption(argv, "--stratify-by")
+      });
+      return printJson(hasFlag(argv, "--dry-run") ? prepared.preview : await executeDatasetSplit(cwd(), prepared));
+    }
     const inputPath = resolve3(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH);
-    const config = await configFromArgv(argv, inputPath);
-    const input = await loadCliBehaviorSpec(inputPath);
+    const config = readOption(argv, "--repo") ? await resolveDownloadConfig(inputPath, readOption(argv, "--config"), (message) => console.error(message)) : await configFromArgv(argv, inputPath);
     if (subcommand !== "prefetch" && subcommand !== "verify") throw new Error(`Unknown datasets command: ${subcommand}`);
+    const reporter = createConsoleReporter({
+      verbose: hasFlag(argv, "--verbose"),
+      quiet: hasFlag(argv, "--quiet")
+    });
+    const source = datasetSourceFromArgv(argv);
+    if (source) {
+      if (subcommand !== "prefetch") throw new Error("--repo applies to `tt datasets prefetch`; use `tt assets verify` for downloads.");
+      return printJson(await prefetchHuggingFaceDataset({ request: { dataset_prebuilt: source }, config, reporter }));
+    }
+    const input = await loadCliBehaviorSpec(inputPath);
     const report = await prefetchHuggingFaceDataset({
-      request: input.request,
+      request: applySpecLock(input.request, await readSpecLock(input.path)),
       config,
       localOnly: subcommand === "verify",
-      reporter: createConsoleReporter({
-        verbose: hasFlag(argv, "--verbose"),
-        quiet: hasFlag(argv, "--quiet")
-      })
+      reporter
     });
-    return printJson({ ...report, input_path: input.path });
+    if (subcommand === "prefetch") {
+      await writeSpecLockEntry(input.path, "dataset", {
+        repo: report.repo,
+        revision: report.revision,
+        files: (await snapshotDigests(report.snapshot_path, Object.values(report.files).flatMap((file) => file ? [file.path] : []))).files
+      });
+    }
+    return printJson({ ...report, input_path: input.path, ...subcommand === "prefetch" ? { lock_path: specLockPath(input.path) } : {} });
+  }
+  if (command === "assets") {
+    const subcommand = cli.subcommand;
+    if (subcommand === "list") return printJson(await listAssets());
+    if (subcommand === "path") {
+      const asset = await findAsset(cli.positionals[0]);
+      console.log(asset.snapshot_path);
+      return;
+    }
+    if (subcommand === "verify") {
+      const assets = cli.positionals[0] ? [await findAsset(cli.positionals[0])] : await listAssets();
+      const results = [];
+      for (const asset of assets) {
+        results.push({ kind: asset.kind, repo: asset.repo, revision: asset.revision, ...await verifyAsset(asset) });
+      }
+      printJson({ ok: results.every((result) => result.ok), assets: results });
+      if (!results.every((result) => result.ok)) process.exitCode = 2;
+      return;
+    }
+    throw new Error(`Unknown assets command: ${subcommand}`);
   }
   if (command === "models") {
     const subcommand = cli.subcommand;
     const modelInputPath = subcommand === "prefetch" || subcommand === "verify-base" ? resolve3(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH) : void 0;
-    const config = await configFromArgv(argv, modelInputPath);
+    const config = subcommand === "prefetch" && readOption(argv, "--repo") ? await resolveDownloadConfig(modelInputPath, readOption(argv, "--config"), (message) => console.error(message)) : await configFromArgv(argv, modelInputPath);
     const store = createLocalStore(config.storeRoot);
     if (subcommand === "list") return printJson(await store.listModels());
     if (subcommand === "get") {
@@ -2040,19 +2178,35 @@ ${validation.errors.join("\n")}`);
       return printJson({ active: pointer.model_id ?? "base", pointer });
     }
     if (subcommand === "prefetch" || subcommand === "verify-base") {
+      const reporter = createConsoleReporter({
+        verbose: hasFlag(argv, "--verbose"),
+        quiet: hasFlag(argv, "--quiet")
+      });
+      const repo = readOption(argv, "--repo");
+      if (repo) {
+        if (subcommand !== "prefetch") throw new Error("--repo applies to `tt models prefetch`; use `tt assets verify` for downloads.");
+        const revision = readOption(argv, "--revision");
+        return printJson(await prefetchBaseModel({ source: { repo, ...revision ? { revision } : {} }, config, reporter }));
+      }
       const input = await loadCliBehaviorSpec(modelInputPath);
       const report = await prefetchBaseModel({
         request: input.request,
         config,
         localOnly: subcommand === "verify-base",
-        reporter: createConsoleReporter({
-          verbose: hasFlag(argv, "--verbose"),
-          quiet: hasFlag(argv, "--quiet")
-        })
+        reporter
       });
+      const locked = subcommand === "prefetch" && report.snapshot_path && report.snapshot_revision;
+      if (locked) {
+        await writeSpecLockEntry(input.path, "base_model", {
+          repo: report.base_model,
+          revision: report.snapshot_revision,
+          files: (await snapshotDigests(report.snapshot_path)).files
+        });
+      }
       return printJson({
         ...report,
-        input_path: input.path
+        input_path: input.path,
+        ...locked ? { lock_path: specLockPath(input.path) } : {}
       });
     }
     if (subcommand === "serve") {
@@ -2089,6 +2243,7 @@ export {
   buildLocalBaseModelServerLaunch,
   buildLocalModelServerLaunch,
   buildModelPrefetchPayload,
+  buildModelSourcePrefetchPayload,
   buildSystemMessage,
   canonicalLocalPipeline,
   compareRuns,
@@ -2168,6 +2323,7 @@ export {
   verifyLocalBaseModel,
   verifyLocalModelServerLaunch,
   verifyModelPrefetchCacheReport,
+  withProgressHeartbeat,
   writePrivateJsonAtomic
 };
 //# sourceMappingURL=local-runtime.js.map