@tuned-tensor/cli 0.13.1-beta.5 → 0.13.1-beta.6

package/README.md

@@ -141,7 +141,7 @@ guarantee that a model or dataset is compatible with a TT workflow.
 
 For educational questions about how or why training works, the agent can read
 the exact Python implementation shipped with the running TT build. The source
-tool is limited to named foundation components (tokenizer, pretraining, model,
+tool is limited to named foundation components (tokenizer, pretraining, checkpointing, model,
 data, SFT, RL, evaluation, and common helpers) and adapter components (training,
 data, model contract, and evaluation); it cannot read arbitrary paths. Answers
 separate behavior directly visible in the code from inferred rationale and
@@ -206,16 +206,18 @@ Two local engines share that document version:
   `inference` evaluators. Requires a foundation `tunedtensor.json`
   (`engine: "foundation"`, at least two examples, no `base_model`).
 
-The foundation engine is a small, readable local baseline inspired by Andrej
+The foundation engine is a readable, single-GPU local trainer inspired by Andrej
 Karpathy's [nanochat](https://github.com/karpathy/nanochat), not a port of its
-compute-optimal training stack. The first version deliberately builds a bounded
-pretraining corpus from the spec prompt and examples, reports chat accuracy on
-those same contract examples, and supports one training process. Foundation
+distributed training stack. With `foundation.corpus_path`, it streams `.txt`
+or JSONL (`{"text":"..."}`) shards into a deterministic on-disk token cache;
+without that field it retains the bounded prompt/example corpus for smoke runs.
+`foundation.validation_path` supplies a distinct held-out BPB corpus. Foundation
 plans reject `compare` before creating run artifacts; comparison remains an
 adapter-engine capability. Optional RL samples one seeded on-policy completion
 per step with a sparse last-number exact reward and therefore requires a numeric
 expected output for every example. Treat those metrics as an end-to-end
-execution and overfit check—not held-out capability or multi-GPU evidence.
+execution and overfit check—not held-out capability or multi-GPU evidence when
+no validation corpus is configured.
 
 ```bash
 tt pipeline init --file pipeline.json
@@ -227,6 +229,7 @@ tt --json pipeline run --file pipeline.json \
   --spec tunedtensor.json --config local-runner.json
 tt init --engine foundation --name "Tiny GPT"
 tt --json pipeline run --spec tunedtensor.json --dry-run
+tt pipeline run --spec tunedtensor.json --resume /absolute/path/to/foundation-run
 ```
 
 `--only` and `--skip` preserve dependency safety: a selected step cannot refer
@@ -234,6 +237,14 @@ to an omitted predecessor. Cloud-targeted steps fail closed unless you pass
 `--dry-run`. Foundation documents are local-only. The TT agent may describe,
 validate, and prepare a plan, but has no direct pipeline-execute tool.
 
+Long pretraining uses BF16, warmup plus cosine decay, gradient clipping,
+periodic metrics, and atomic rolling checkpoints. `--resume <run-directory>`
+starts or resumes one stable run identity; completed stages are verified and
+skipped, while an interrupted pretrain step restores model, optimizer,
+scheduler, RNG, counters, and token cursor. See
+[Long foundation runs](docs/local-runtime/foundation-long-runs.md) before an
+unattended job.
+
 ## Quick start
 
 Create a local project on an NVIDIA host:

package/dist/{chunk-AVMRHDFY.js → chunk-W7MOBPCO.js}

@@ -1176,7 +1176,7 @@ function warningsFromSnapshot(report, target) {
 }
 
 // src/local-runtime/host-inventory.ts
-import { constants } from "fs";
+import { constants as constants2 } from "fs";
 import { access, statfs } from "fs/promises";
 import { cpus, freemem, homedir as homedir3, totalmem, type } from "os";
 import { dirname as dirname5, join as join5, resolve as resolve7 } from "path";
@@ -1334,8 +1334,8 @@ function withOfflineHuggingFaceCacheEnvironment(env, modelCache) {
 // src/local-runtime/process-runner.ts
 import { createHash as createHash2 } from "crypto";
 import { spawn } from "child_process";
-import { createWriteStream, readFileSync } from "fs";
-import { mkdir as mkdir2 } from "fs/promises";
+import { constants, readFileSync } from "fs";
+import { mkdir as mkdir2, open as open2 } from "fs/promises";
 import { dirname as dirname4, join as join4, resolve as resolve6 } from "path";
 var packageRoot = localRuntimePackageRoot(import.meta.url);
 var bundledProject = join4(packageRoot, "training/adapter");
@@ -1381,6 +1381,7 @@ function withBundledPythonEnvironment(env) {
   };
 }
 function buildFoundationPythonCommand(entrypoint, args = []) {
+  const target = entrypoint === "-c" ? entrypoint : join4(foundationProject, "src", entrypoint);
   const commandArgs = [
     "run",
     "--frozen",
@@ -1388,7 +1389,7 @@ function buildFoundationPythonCommand(entrypoint, args = []) {
     "--project",
     foundationProject,
     "python",
-    join4(foundationProject, "src", entrypoint),
+    target,
     ...args
   ];
   return {
@@ -1409,28 +1410,19 @@ var ProcessCancelledError = class extends Error {
     this.name = "ProcessCancelledError";
   }
 };
-async function openProcessLog(path, exclusive) {
-  const stream = createWriteStream(path, {
-    flags: exclusive ? "wx" : "w",
-    mode: 384
-  });
-  await new Promise((resolveOpen, reject) => {
-    const onOpen = () => {
-      stream.off("error", onError);
-      resolveOpen();
-    };
-    const onError = (error) => {
-      stream.off("open", onOpen);
-      reject(error);
-    };
-    stream.once("open", onOpen);
-    stream.once("error", onError);
-  });
-  return stream;
+async function openProcessLog(path, exclusive, append) {
+  const flags = constants.O_WRONLY | constants.O_CREAT | (exclusive ? constants.O_EXCL : append ? constants.O_APPEND : constants.O_TRUNC) | (process.platform === "win32" ? 0 : constants.O_NOFOLLOW);
+  const handle = await open2(path, flags, 384);
+  try {
+    return handle.createWriteStream({ autoClose: true });
+  } catch (error) {
+    await handle.close();
+    throw error;
+  }
 }
 async function runLoggedProcess(args) {
   if (args.logPath) await mkdir2(dirname4(args.logPath), { recursive: true });
-  const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false) : null;
+  const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false, args.appendLog ?? false) : null;
   let stderr = "";
   try {
     const exitCode = await new Promise((resolvePromise, reject) => {
@@ -1458,7 +1450,7 @@ async function runLoggedProcess(args) {
       const requestStop = (signal = "SIGTERM") => {
         killProcessGroup(signal);
         if (!forceKillTimer) {
-          forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), 5e3);
+          forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), args.shutdownGraceMs ?? 5e3);
           forceKillTimer.unref();
         }
       };
@@ -1705,7 +1697,7 @@ async function queryNvidiaGpus(env) {
 async function diskCheck(name, path) {
   const resolvedPath = resolve7(path);
   try {
-    await access(resolvedPath, constants.R_OK);
+    await access(resolvedPath, constants2.R_OK);
     const fs = await statfs(resolvedPath);
     const freeBytes = Number(fs.bavail) * Number(fs.bsize);
     return {
@@ -2538,14 +2530,30 @@ var specSnapshotSchema = z.object({
 }).strict();
 var foundationHyperparametersSchema = z.object({
   vocab_size: z.number().int().min(64).max(65536).default(256),
-  max_chars: z.number().int().min(1e3).max(2e9).default(2e4),
+  max_chars: z.number().int().min(1e3).max(1e12).default(2e4),
   depth: z.number().int().min(1).max(64).default(2),
   pretrain_steps: z.number().int().min(1).max(1e7).default(2),
   finetune_steps: z.number().int().min(1).max(1e7).default(2),
   rl_steps: z.number().int().min(0).max(1e7).default(0),
   batch_size: z.number().int().min(1).max(1024).default(2),
   sequence_length: z.number().int().min(16).max(8192).default(64),
-  nproc_per_node: z.number().int().min(1).max(16).default(1)
+  nproc_per_node: z.number().int().min(1).max(16).default(1),
+  corpus_path: z.string().min(1).optional(),
+  tokenizer_max_chars: z.number().int().min(1e3).max(1e12).optional(),
+  validation_path: z.string().min(1).optional(),
+  checkpoint_backup_dir: z.string().min(1).optional(),
+  seed: z.number().int().min(0).max(2147483647).optional(),
+  learning_rate: z.number().positive().max(1).optional(),
+  weight_decay: z.number().min(0).max(10).optional(),
+  warmup_steps: z.number().int().min(0).max(1e7).optional(),
+  min_lr_ratio: z.number().min(0).max(1).optional(),
+  gradient_accumulation_steps: z.number().int().min(1).max(1024).optional(),
+  gradient_clip: z.number().positive().max(1e3).optional(),
+  bf16: z.boolean().optional(),
+  checkpoint_interval_steps: z.number().int().min(1).max(1e7).optional(),
+  checkpoint_interval_seconds: z.number().int().min(1).max(86400).optional(),
+  keep_checkpoints: z.number().int().min(1).max(100).optional(),
+  log_interval_steps: z.number().int().min(1).max(1e6).optional()
 }).strict();
 var localFoundationSpecFileSchema = z.object({
   engine: z.literal("foundation"),
@@ -2799,7 +2807,7 @@ function isFoundationSpecFile(spec) {
 // src/local-runtime/local-project.ts
 import { createHash as createHash3, randomUUID as randomUUID3 } from "crypto";
 import { readFile as readFile4, stat as stat3, writeFile as writeFile4 } from "fs/promises";
-import { dirname as dirname8, isAbsolute as isAbsolute3, resolve as resolve9 } from "path";
+import { dirname as dirname8, isAbsolute as isAbsolute3, relative as relative2, resolve as resolve9, sep as sep3 } from "path";
 import { homedir as homedir4 } from "os";
 import { mkdir as mkdir5 } from "fs/promises";
 var DEFAULT_LOCAL_SPEC_PATH = "tunedtensor.json";
@@ -2811,6 +2819,13 @@ function resolveLocalReference(value, baseDirectory) {
   if (value.startsWith("~/")) return resolve9(homedir4(), value.slice(2));
   return resolve9(baseDirectory, value);
 }
+function localPathsOverlap(left, right) {
+  const contains = (parent, candidate) => {
+    const relation = relative2(resolve9(parent), resolve9(candidate));
+    return relation === "" || !isAbsolute3(relation) && relation !== ".." && !relation.startsWith(`..${sep3}`);
+  };
+  return contains(left, right) || contains(right, left);
+}
 function resolveLocalRunInputPaths(raw, inputPath) {
   if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw;
   const baseDirectory = dirname8(resolve9(inputPath));
@@ -2822,6 +2837,13 @@ function resolveLocalRunInputPaths(raw, inputPath) {
       if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
     }
   }
+  const foundation = value.foundation;
+  if (foundation && typeof foundation === "object" && !Array.isArray(foundation)) {
+    const fields = foundation;
+    for (const key of ["corpus_path", "validation_path", "checkpoint_backup_dir"]) {
+      if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
+    }
+  }
   return value;
 }
 async function exists2(path) {
@@ -3138,7 +3160,7 @@ async function evaluationSuiteFromChatJsonl(path, configuredSystem) {
 import { createHash as createHash4 } from "crypto";
 import { createReadStream as createReadStream2 } from "fs";
 import { lstat as lstat4, mkdir as mkdir6, readFile as readFile6, readdir as readdir4, realpath as realpath2, stat as stat4 } from "fs/promises";
-import { isAbsolute as isAbsolute4, join as join8, relative as relative2, resolve as resolve10, sep as sep3 } from "path";
+import { isAbsolute as isAbsolute4, join as join8, relative as relative3, resolve as resolve10, sep as sep4 } from "path";
 var NEMOTRON_BF16_FILE_DIGESTS = {
   ".gitattributes": { algorithm: "sha1", digest: "fe3f694ce14b577c9833fca6e83c5e26a7119127" },
   "LICENSE": { algorithm: "sha1", digest: "4d76cc87c9edb280753e3bee3b499ae461113835" },
@@ -3204,8 +3226,8 @@ async function certifiedFileDigest(path, expected) {
   return digest.digest("hex");
 }
 function isStrictPhysicalDescendant(parent, child) {
-  const candidate = relative2(parent, child);
-  return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep3}`) && !isAbsolute4(candidate);
+  const candidate = relative3(parent, child);
+  return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep4}`) && !isAbsolute4(candidate);
 }
 async function verifySnapshotCacheLocation(path, modelId, revision, modelCache) {
   const repository = `models--${modelId.replaceAll("/", "--")}`;
@@ -3297,7 +3319,7 @@ async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelC
     }
     physicalFiles.set(file, target);
   }
-  const actualFiles = new Set(snapshotFiles.map((file) => relative2(path, file).split("\\").join("/")));
+  const actualFiles = new Set(snapshotFiles.map((file) => relative3(path, file).split("\\").join("/")));
   const expectedFiles = new Set(Object.keys(digests));
   const unexpected = [...actualFiles].filter((name) => !expectedFiles.has(name)).sort();
   if (unexpected.length > 0) {
@@ -3578,12 +3600,12 @@ function evaluateGeneralRegressionGate(comparison, policy) {
 }
 
 // src/local-runtime/orchestrator.ts
-import { lstat as lstat5, mkdir as mkdir9, open as open2, readFile as readFile9, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
+import { lstat as lstat5, mkdir as mkdir9, open as open3, readFile as readFile9, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
 import { createReadStream as createReadStream3 } from "fs";
 import { createHash as createHash6, randomUUID as randomUUID4 } from "crypto";
 import { performance } from "perf_hooks";
 import { homedir as homedir5 } from "os";
-import { basename, dirname as dirname10, join as join11, relative as relative3, resolve as resolve13 } from "path";
+import { basename, dirname as dirname10, join as join11, relative as relative4, resolve as resolve13 } from "path";
 
 // src/local-runtime/evaluation.ts
 import { createHash as createHash5 } from "crypto";
@@ -4678,7 +4700,7 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
       }
       if (!childMetadata.isFile()) continue;
       files.push({
-        path: relative3(root, child).split("\\").join("/"),
+        path: relative4(root, child).split("\\").join("/"),
         size_bytes: childMetadata.size,
         sha256: await hashFile(child)
       });
@@ -4761,7 +4783,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
   await removeReport();
 }
 function manifestRelativePath(artifacts, path) {
-  return relative3(artifacts.runDir, path).split("\\").join("/");
+  return relative4(artifacts.runDir, path).split("\\").join("/");
 }
 async function verifyReusableArtifacts(artifacts, paths, options = {}) {
   if (!await pathExists2(artifacts.artifactManifestJson)) return false;
@@ -4782,7 +4804,7 @@ async function acquireWorkflowLock(lockPath, description) {
   const token = randomUUID4();
   for (let attempt = 0; attempt < 3; attempt += 1) {
     try {
-      const handle = await open2(lockPath, "wx", 384);
+      const handle = await open3(lockPath, "wx", 384);
       try {
         await handle.writeFile(`${JSON.stringify({ token, pid: process.pid, created_at: (/* @__PURE__ */ new Date()).toISOString() })}
 `);
@@ -5773,6 +5795,7 @@ export {
   localRunnerConfigSchema,
   isFoundationSpecFile,
   DEFAULT_LOCAL_SPEC_PATH,
+  localPathsOverlap,
   resolveLocalRunInputPaths,
   initLocalSpecFile,
   initLocalRunnerConfigFile,
@@ -5805,4 +5828,4 @@ export {
   runLocalPipeline,
   runLocalFineTune
 };
-//# sourceMappingURL=chunk-AVMRHDFY.js.map
+//# sourceMappingURL=chunk-W7MOBPCO.js.map