@tuned-tensor/cli 0.13.1-beta.5 → 0.13.1-beta.7

package/dist/local-runtime.js

@@ -8,7 +8,9 @@ import {
   assessHardware,
   baseModelRevisionSchema,
   behaviorSpecExampleSchema,
+  behaviorSpecSchema,
   buildBundledPythonCommand,
+  buildFoundationPythonCommand,
   buildModelPrefetchPayload,
   buildSystemMessage,
   canonicalLocalPipeline,
@@ -47,6 +49,7 @@ import {
   localAdapterSpecFileSchema,
   localBehaviorSpecFileSchema,
   localFoundationSpecFileSchema,
+  localPathsOverlap,
   localRunnerConfigSchema,
   localRuntimePackageRoot,
   localStorePaths,
@@ -71,10 +74,11 @@ import {
   verifyLocalBaseModel,
   verifyModelPrefetchCacheReport,
   withBundledPythonEnvironment,
+  withFoundationPythonEnvironment,
   withHuggingFaceCacheEnvironment,
   withOfflineHuggingFaceCacheEnvironment,
   writePrivateJsonAtomic
-} from "./chunk-AVMRHDFY.js";
+} from "./chunk-AKJ5WZ2S.js";
 
 // src/local-runtime/index.ts
 import { readFileSync, realpathSync } from "fs";
@@ -146,7 +150,7 @@ function compareRuns(a, b) {
 
 // src/local-runtime/doctor.ts
 import { constants } from "fs";
-import { access, mkdir, statfs, unlink, writeFile } from "fs/promises";
+import { access, lstat, mkdir, statfs, unlink, writeFile } from "fs/promises";
 import { homedir } from "os";
 import { join, resolve } from "path";
 import { spawn } from "child_process";
@@ -266,6 +270,23 @@ async function writableDirectoryCheck(name, path) {
     };
   }
 }
+function foundationPythonProbePlan(requireBf16) {
+  const source = [
+    "import json",
+    "import numpy, safetensors, tokenizers, torch",
+    "assert torch.cuda.is_available(), 'TT Foundation training requires CUDA but torch.cuda.is_available() is false'",
+    ...requireBf16 ? ["assert torch.cuda.is_bf16_supported(), 'TT Foundation BF16 training was requested but this device does not support BF16'"] : [],
+    "props = torch.cuda.get_device_properties(0)",
+    "print(json.dumps({'python_ok': True, 'torch': torch.__version__, 'cuda_available': True, 'bf16': torch.cuda.is_bf16_supported(), 'cuda_device': torch.cuda.get_device_name(0), 'compute_capability': list(torch.cuda.get_device_capability(0)), 'total_memory_bytes': props.total_memory}))"
+  ].join("; ");
+  const entrypoint = buildFoundationPythonCommand("-c", [source]);
+  return {
+    name: "python-runtime",
+    command: entrypoint.command,
+    args: entrypoint.commandArgs,
+    env: withFoundationPythonEnvironment(process.env)
+  };
+}
 function pythonProbeSource(device) {
   return [
     "import json",
@@ -311,6 +332,26 @@ function foundationPlaceholderCheck(spec) {
     message: issues.length > 0 ? "The spec still contains generated placeholder content; edit it before training." : `${spec.examples.length} spec example(s); foundation engine`
   };
 }
+async function foundationCorpusCheck(name, path) {
+  const resolvedPath = resolve(path);
+  try {
+    const info = await lstat(resolvedPath);
+    const supported = info.isFile() || info.isDirectory();
+    return {
+      name,
+      ok: supported && !info.isSymbolicLink(),
+      message: supported && !info.isSymbolicLink() ? `${resolvedPath} is available for streaming.` : `${resolvedPath} must be a regular file or directory, not a symbolic link.`,
+      details: { path: resolvedPath }
+    };
+  } catch (error) {
+    return {
+      name,
+      ok: false,
+      message: `${resolvedPath}: ${error instanceof Error ? error.message : String(error)}`,
+      details: { path: resolvedPath }
+    };
+  }
+}
 function placeholderSpecCheck(request) {
   const placeholder = request.spec_snapshot.examples.some(
     (example) => /replace this with/i.test(example.input) || /replace this with/i.test(example.output)
@@ -360,11 +401,33 @@ async function runDoctor(config, request, foundationSpec) {
   }
   if (foundationSpec) {
     checks.push(foundationPlaceholderCheck(foundationSpec));
+    if (foundationSpec.foundation.corpus_path) {
+      checks.push(await foundationCorpusCheck("pretraining-corpus", foundationSpec.foundation.corpus_path));
+    }
+    if (foundationSpec.foundation.validation_path) {
+      checks.push(await foundationCorpusCheck("validation-corpus", foundationSpec.foundation.validation_path));
+    }
+    if (foundationSpec.foundation.corpus_path && foundationSpec.foundation.validation_path && localPathsOverlap(
+      foundationSpec.foundation.corpus_path,
+      foundationSpec.foundation.validation_path
+    )) {
+      checks.push({
+        name: "held-out-corpus",
+        ok: false,
+        message: "foundation.corpus_path and foundation.validation_path must be distinct."
+      });
+    }
+    if (foundationSpec.foundation.checkpoint_backup_dir) {
+      checks.push(await writableDirectoryCheck(
+        "checkpoint-backup",
+        foundationSpec.foundation.checkpoint_backup_dir
+      ));
+    }
   } else if (request) {
     checks.push(placeholderSpecCheck(request));
     resolveTrainingModel(request.spec_snapshot.base_model);
   }
-  const pythonPlans = buildDoctorPythonPlans(config);
+  const pythonPlans = foundationSpec && !config.dryRun ? [foundationPythonProbePlan(foundationSpec.foundation.bf16 !== false)] : buildDoctorPythonPlans(config);
   if (pythonPlans.length > 0) {
     const uvVersion = await runCommand("uv", ["--version"]);
     checks.push({
@@ -417,20 +480,20 @@ async function runDoctor(config, request, foundationSpec) {
       message: config.dryRun ? "GPU checks skipped because dryRun is enabled." : `nvidia-smi is not required for device=${device}.`
     });
   }
-  const trainingCommand = buildBundledPythonCommand("train.py").displayCommand;
+  const trainingCommand = foundationSpec ? buildFoundationPythonCommand("pretrain.py").displayCommand : buildBundledPythonCommand("train.py").displayCommand;
   checks.push({
     name: "effective-plan",
     ok: true,
     message: "Resolved the configured training and evaluation plan.",
     details: {
       training_command: trainingCommand ?? null,
-      evaluation_provider: "transformers",
+      evaluation_provider: foundationSpec ? "foundation" : "transformers",
       evaluation_device: device,
       artifact_root: resolve(config.artifactRoot),
       store_root: resolve(config.storeRoot ?? defaultLocalHome()),
       model_cache: resolve(config.paths.modelCache ?? process.env.HF_HOME ?? join(homedir(), ".cache", "huggingface")),
       base_model: request?.spec_snapshot.base_model ?? null,
-      scoring_mode: config.evaluation.scoring.mode
+      scoring_mode: foundationSpec ? "bpb/chat" : config.evaluation.scoring.mode
     }
   });
   return checks;
@@ -1746,6 +1809,7 @@ export {
   assertLocalRunInputReady,
   baseModelRevisionSchema,
   behaviorSpecExampleSchema,
+  behaviorSpecSchema,
   buildLocalBaseModelServerLaunch,
   buildLocalModelServerLaunch,
   buildModelPrefetchPayload,
@@ -1788,6 +1852,7 @@ export {
   localAdapterSpecFileSchema,
   localBehaviorSpecFileSchema,
   localFoundationSpecFileSchema,
+  localPathsOverlap,
   localRunnerConfigSchema,
   localStorePaths,
   normalizeChatJsonlForRelocation,