@tuned-tensor/cli 0.13.1-beta.5 → 0.13.1-beta.6

package/dist/index.js

@@ -16,12 +16,13 @@ import {
   hardenExistingLocalStore,
   loadLocalRunInput,
   loadLocalRunnerConfig,
+  localPathsOverlap,
   readHardwareSnapshot,
   runLocalPipeline,
   runLoggedProcess,
   warningsFromSnapshot,
   withFoundationPythonEnvironment
-} from "./chunk-AVMRHDFY.js";
+} from "./chunk-W7MOBPCO.js";
 
 // src/cli.ts
 import { spawn as spawn2 } from "child_process";
@@ -1067,6 +1068,7 @@ var TRAINING_SOURCE_FILES = {
   foundation: {
     tokenizer: "training/foundation/src/train_tokenizer.py",
     pretrain: "training/foundation/src/pretrain.py",
+    checkpoint: "training/foundation/src/checkpoint.py",
     model: "training/foundation/src/model.py",
     data: "training/foundation/src/data.py",
     finetune: "training/foundation/src/finetune.py",
@@ -1522,6 +1524,7 @@ function createTunedTensorTools(api, options = {}) {
           component: Type2.Union([
             Type2.Literal("tokenizer"),
             Type2.Literal("pretrain"),
+            Type2.Literal("checkpoint"),
             Type2.Literal("model"),
             Type2.Literal("data"),
             Type2.Literal("finetune"),
@@ -5134,8 +5137,8 @@ import { resolve as resolve5 } from "path";
 // src/local-runtime/foundation-runner.ts
 import { createHash as createHash3, randomUUID as randomUUID5 } from "crypto";
 import { createReadStream } from "fs";
-import { chmod as chmod2, lstat as lstat3, mkdir as mkdir3, open as open2, readdir as readdir3, readFile as readFile4 } from "fs/promises";
-import { dirname as dirname4, join as join7, resolve as resolve4 } from "path";
+import { chmod as chmod2, lstat as lstat3, mkdir as mkdir3, open as open2, readdir as readdir3, readFile as readFile4, rename as rename2, rm as rm2 } from "fs/promises";
+import { basename as basename4, dirname as dirname4, join as join7, resolve as resolve4 } from "path";
 function isRef(value) {
   return Boolean(value) && typeof value === "object" && !Array.isArray(value) && typeof value.from === "string";
 }
@@ -5204,13 +5207,20 @@ async function readRequiredJsonObject(path, description) {
   }
   return parsed;
 }
-async function writePrivateText(path, content) {
-  const file = await open2(path, "w", 384);
+async function writePrivateTextAtomic(path, content) {
+  const temporary = join7(dirname4(path), `.${basename4(path)}.${randomUUID5()}.tmp`);
+  const file = await open2(temporary, "wx", 384);
+  let closed = false;
   try {
     await file.chmod(384);
     await file.writeFile(content, "utf8");
-  } finally {
+    await file.sync();
     await file.close();
+    closed = true;
+    await rename2(temporary, path);
+  } finally {
+    if (!closed) await file.close().catch(() => void 0);
+    await rm2(temporary, { force: true }).catch(() => void 0);
   }
 }
 async function artifactFor(path) {
@@ -5247,6 +5257,11 @@ function assertFoundationPlanSupported(spec, plan) {
       `Foundation runner does not implement ${unsupported.uses} (step ${unsupported.id}).`
     );
   }
+  if (spec.foundation.corpus_path && spec.foundation.validation_path && localPathsOverlap(spec.foundation.corpus_path, spec.foundation.validation_path)) {
+    throw new Error(
+      "Foundation training and validation paths overlap; use disjoint corpora for held-out evaluation."
+    );
+  }
   if (plan.steps.some((step) => step.uses === "rl")) {
     const invalidReward = spec.examples.findIndex(
       (example) => !/-?\d+(?:\.\d+)?/.test(example.output.replaceAll(",", ""))
@@ -5274,7 +5289,9 @@ async function defaultFoundationSpawn(args2) {
     commandArgs: launched.commandArgs,
     env: withFoundationPythonEnvironment(process.env),
     logPath: args2.logPath,
-    stage: args2.stepId
+    stage: args2.stepId,
+    shutdownGraceMs: 12e4,
+    appendLog: true
   });
   if (result2.exitCode !== 0) {
     throw new Error(
@@ -5288,8 +5305,26 @@ async function runFoundationPipeline(args2) {
   const outputDir = resolve4(
     args2.outputDir ?? join7(dirname4(resolve4(args2.specPath)), DEFAULT_FOUNDATION_RUNS_DIR, randomUUID5())
   );
-  await mkdir3(outputDir, { recursive: true, mode: 448 });
-  await chmod2(outputDir, 448);
+  let outputExists = false;
+  try {
+    const outputInfo = await lstat3(outputDir);
+    if (!args2.resume) {
+      throw new Error(`Foundation output directory already exists; pass --resume ${outputDir} to recover it.`);
+    }
+    if (outputInfo.isSymbolicLink()) {
+      throw new Error(`Foundation resume directory must not be a symbolic link: ${outputDir}`);
+    }
+    if (!outputInfo.isDirectory()) {
+      throw new Error(`Foundation resume path must be a directory: ${outputDir}`);
+    }
+    outputExists = true;
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error;
+  }
+  if (!outputExists) {
+    await mkdir3(outputDir, { recursive: true, mode: 448 });
+  }
+  await makePrivateTree(outputDir);
   const spawnStep = args2.spawnStep ?? defaultFoundationSpawn;
   const produced = /* @__PURE__ */ new Map();
   const steps = [];
@@ -5297,6 +5332,7 @@ async function runFoundationPipeline(args2) {
   const hp = args2.spec.foundation;
   let tokenizerDir;
   let modelDir;
+  let upstreamExecuted = false;
   const resolveKind = (stepId, kind) => {
     const path = produced.get(stepId)?.[kind];
     if (!path) throw new Error(`Step ${stepId} did not produce ${kind}.`);
@@ -5316,6 +5352,8 @@ async function runFoundationPipeline(args2) {
     await chmod2(output, 448);
     const configPath2 = join7(stepDir, "config.json");
     const logPath = join7(stepDir, "step.log");
+    const metricsPath = join7(output, "metrics.json");
+    const completionPath = join7(stepDir, "completion.json");
     const fields = stepConfig(step);
     let entrypoint;
     let config;
@@ -5324,7 +5362,8 @@ async function runFoundationPipeline(args2) {
       config = {
         output_dir: output,
         vocab_size: fields.vocabSize ?? hp.vocab_size,
-        max_chars: fields.maxChars ?? hp.max_chars,
+        max_chars: hp.tokenizer_max_chars ?? fields.maxChars ?? hp.max_chars,
+        corpus_path: hp.corpus_path,
         system_prompt: args2.spec.system_prompt,
         examples
       };
@@ -5334,12 +5373,28 @@ async function runFoundationPipeline(args2) {
       config = {
         tokenizer_dir: tokenizerDir,
         output_dir: output,
+        work_dir: join7(stepDir, "recovery"),
         depth: fields.depth ?? hp.depth,
         steps: fields.steps ?? hp.pretrain_steps,
         batch_size: fields.batchSize ?? hp.batch_size,
         sequence_length: fields.sequenceLength ?? hp.sequence_length,
         nproc_per_node: fields.nprocPerNode ?? hp.nproc_per_node,
         max_chars: hp.max_chars,
+        corpus_path: hp.corpus_path,
+        seed: hp.seed,
+        learning_rate: hp.learning_rate,
+        weight_decay: hp.weight_decay,
+        warmup_steps: hp.warmup_steps,
+        min_lr_ratio: hp.min_lr_ratio,
+        gradient_accumulation_steps: hp.gradient_accumulation_steps,
+        gradient_clip: hp.gradient_clip,
+        bf16: hp.bf16,
+        checkpoint_interval_steps: hp.checkpoint_interval_steps,
+        checkpoint_interval_seconds: hp.checkpoint_interval_seconds,
+        keep_checkpoints: hp.keep_checkpoints,
+        log_interval_steps: hp.log_interval_steps,
+        checkpoint_backup_dir: hp.checkpoint_backup_dir,
+        resume: true,
         system_prompt: args2.spec.system_prompt,
         examples
       };
@@ -5377,6 +5432,7 @@ async function runFoundationPipeline(args2) {
         output_dir: output,
         sequence_length: hp.sequence_length,
         max_chars: hp.max_chars,
+        validation_path: hp.validation_path,
         system_prompt: args2.spec.system_prompt,
         examples
       };
@@ -5386,12 +5442,33 @@ async function runFoundationPipeline(args2) {
     if ((step.uses === "finetune" || step.uses === "rl" || step.uses === "evaluate") && !tokenizerDir) {
       throw new Error(`Step ${step.id} needs a tokenizer from an earlier tokenize step.`);
     }
-    await writePrivateText(configPath2, `${JSON.stringify(config, null, 2)}
+    let completedStep = false;
+    if (args2.resume) {
+      try {
+        const previousConfig = JSON.parse(await readFile4(configPath2, "utf8"));
+        if (JSON.stringify(previousConfig) !== JSON.stringify(config)) {
+          throw new Error(
+            `Foundation step "${step.id}" configuration changed; resume requires the original spec and pipeline.`
+          );
+        }
+        if (!upstreamExecuted) {
+          await lstat3(metricsPath);
+          await lstat3(completionPath);
+          completedStep = true;
+        }
+      } catch (error) {
+        if (error.code !== "ENOENT") throw error;
+      }
+    }
+    if (!completedStep) {
+      await writePrivateTextAtomic(configPath2, `${JSON.stringify(config, null, 2)}
 `);
-    await spawnStep({ entrypoint, configPath: configPath2, logPath, stepId: step.id });
+      await spawnStep({ entrypoint, configPath: configPath2, logPath, stepId: step.id });
+      upstreamExecuted = true;
+    }
     await makePrivateTree(stepDir);
     const metrics = await readRequiredJsonObject(
-      join7(output, "metrics.json"),
+      metricsPath,
       `Foundation step "${step.id}" metrics`
     );
     if (metrics.ok !== true) {
@@ -5434,6 +5511,26 @@ async function runFoundationPipeline(args2) {
       record4.report = reportPath2;
       artifacts.report = await artifactFor(reportPath2);
     }
+    const completion = {
+      version: 1,
+      config_sha256: await hashPath(configPath2),
+      metrics_sha256: await hashPath(metricsPath),
+      artifacts
+    };
+    if (completedStep) {
+      const previousCompletion = await readRequiredJsonObject(
+        completionPath,
+        `Foundation step "${step.id}" completion manifest`
+      );
+      if (JSON.stringify(previousCompletion) !== JSON.stringify(completion)) {
+        throw new Error(
+          `Foundation step "${step.id}" integrity changed after completion; use a new run directory.`
+        );
+      }
+    } else {
+      await writePrivateTextAtomic(completionPath, `${JSON.stringify(completion, null, 2)}
+`);
+    }
     produced.set(step.id, record4);
     steps.push({ id: step.id, uses: step.uses, metrics, artifacts });
   }
@@ -5444,7 +5541,7 @@ async function runFoundationPipeline(args2) {
     steps
   };
   const reportPath = join7(outputDir, "report.json");
-  await writePrivateText(reportPath, `${JSON.stringify(report, null, 2)}
+  await writePrivateTextAtomic(reportPath, `${JSON.stringify(report, null, 2)}
 `);
   return {
     status: "succeeded",
@@ -5592,7 +5689,10 @@ function registerPipelineCommands(parent) {
     const plan = createExecutionPlan(document, { only: parseList(options.only), skip: parseList(options.skip) });
     outputPlan(plan, await hostWarningsForPipeline(document, options.spec));
   });
-  pipeline.command("run").description("Run an ordered local pipeline, or safely preview any pipeline").option("--dry-run", "Resolve and display only; never execute, transfer, or reserve credits").option("-f, --file <path>", "Pipeline file", DEFAULT_PIPELINE_FILE).option("--spec <path>", "Local behavior spec", DEFAULT_SPEC_FILE).option("--config <path>", "Local runtime config").option("--only <ids>", "Comma-separated step IDs to include").option("--skip <ids>", "Comma-separated step IDs to omit").action(async (options) => {
+  pipeline.command("run").description("Run an ordered local pipeline, or safely preview any pipeline").option("--dry-run", "Resolve and display only; never execute, transfer, or reserve credits").option("-f, --file <path>", "Pipeline file", DEFAULT_PIPELINE_FILE).option("--spec <path>", "Local behavior spec", DEFAULT_SPEC_FILE).option("--config <path>", "Local runtime config").option("--output <path>", "Foundation run directory (must not already exist)").option("--resume <path>", "Resume a foundation run directory").option("--only <ids>", "Comma-separated step IDs to include").option("--skip <ids>", "Comma-separated step IDs to omit").action(async (options) => {
+    if (options.output && options.resume) {
+      throw new Error("--output and --resume are mutually exclusive.");
+    }
     const document = await resolvePipelineDocument(options);
     const plan = createExecutionPlan(document, { only: parseList(options.only), skip: parseList(options.skip) });
     const hostWarnings = await hostWarningsForPipeline(document, options.spec);
@@ -5621,12 +5721,17 @@ function registerPipelineCommands(parent) {
       const result3 = await runFoundationPipeline({
         spec: input.spec,
         plan,
-        specPath: resolve5(options.spec)
+        specPath: resolve5(options.spec),
+        ...options.output || options.resume ? { outputDir: resolve5(options.resume ?? options.output) } : {},
+        resume: Boolean(options.resume)
       });
       if (isJsonMode()) return printJson(result3);
       printSuccess(`Foundation pipeline completed. Report: ${result3.report_path}`);
       return;
     }
+    if (options.output || options.resume) {
+      throw new Error("--output and --resume are only valid for foundation pipelines.");
+    }
     assertLocalRunInputReady(input.request);
     const config = await loadLocalRunnerConfig(options.config ? resolve5(options.config) : void 0);
     const localPipeline = {
@@ -6036,7 +6141,7 @@ async function runCli(version, runtime = {}) {
 }
 
 // src/index.ts
-runCli("0.13.1-beta.5").catch((err) => {
+runCli("0.13.1-beta.6").catch((err) => {
   reportError(err);
   process.exit(1);
 });