@tuned-tensor/cli 0.13.1 → 0.15.0

package/dist/local-runtime.js

@@ -58,6 +58,7 @@ import {
   parseLocalRunInput,
   parseLocalRunnerConfig,
   prefetchBaseModel,
+  pythonEnvironmentPath,
   reportInBackground,
   resolveLocalRunInputPaths,
   resolveModelLoader,
@@ -80,13 +81,13 @@ import {
   withHuggingFaceCacheEnvironment,
   withOfflineHuggingFaceCacheEnvironment,
   writePrivateJsonAtomic
-} from "./chunk-CAEBS5HH.js";
+} from "./chunk-YGN26POJ.js";
 
 // src/local-runtime/index.ts
-import { readFileSync, realpathSync } from "fs";
-import { createHash } from "crypto";
+import { readFileSync as readFileSync2, realpathSync } from "fs";
+import { createHash as createHash2 } from "crypto";
 import { readFile as readFile2, stat } from "fs/promises";
-import { basename, dirname, join as join3, resolve as resolve3 } from "path";
+import { basename, dirname, join as join4, resolve as resolve3 } from "path";
 import { cwd } from "process";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
@@ -294,7 +295,7 @@ function pythonProbeSource(device) {
     "import json",
     "import torch, transformers, peft, huggingface_hub",
     `requested = ${JSON.stringify(device)}`,
-    "assert torch.cuda.is_available(), 'TT Local training requires CUDA but torch.cuda.is_available() is false'",
+    "assert torch.cuda.is_available(), 'TT local training requires CUDA but torch.cuda.is_available() is false'",
     "props = torch.cuda.get_device_properties(0)",
     "print(json.dumps({'python_ok': True, 'torch': torch.__version__, 'transformers': transformers.__version__, 'cuda_available': True, 'cuda_device': torch.cuda.get_device_name(0), 'compute_capability': list(torch.cuda.get_device_capability(0)), 'total_memory_bytes': props.total_memory, 'evaluation_device': requested}))"
   ].join("; ");
@@ -505,10 +506,43 @@ async function runDoctor(config, request, foundationSpec) {
 import { spawn as spawn2 } from "child_process";
 import { resolve as resolve2 } from "path";
 import { fileURLToPath } from "url";
+
+// src/local-runtime/serving-runtime.ts
+import { createHash } from "crypto";
+import { readFileSync } from "fs";
+import { join as join2 } from "path";
+var root = localRuntimePackageRoot(import.meta.url);
+var project = join2(root, "training/serving");
+var hash = createHash("sha256");
+for (const name of ["pyproject.toml", "uv.lock"]) hash.update(readFileSync(join2(project, name)));
+var SERVING_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-serving", hash.digest("hex").slice(0, 20));
+function buildServingPythonCommand() {
+  const commandArgs = [
+    "run",
+    "--frozen",
+    "--quiet",
+    "--project",
+    project,
+    "python",
+    join2(root, "training/adapter/src/serve.py")
+  ];
+  return { command: "uv", commandArgs, displayCommand: ["uv", ...commandArgs] };
+}
+function withServingPythonEnvironment(env) {
+  return {
+    ...env,
+    UV_PROJECT_ENVIRONMENT: SERVING_PYTHON_ENVIRONMENT,
+    VLLM_NO_USAGE_STATS: "1",
+    DO_NOT_TRACK: "1",
+    VLLM_ALLOW_RUNTIME_LORA_UPDATING: "0"
+  };
+}
+
+// src/local-runtime/model-server.ts
 function localArtifactPath(uri) {
   if (uri.startsWith("file://")) return fileURLToPath(uri);
   if (/^[a-z][a-z0-9+.-]*:/i.test(uri)) {
-    throw new Error(`TT Local serving requires a local file artifact, got: ${uri}`);
+    throw new Error(`Local TT serving requires a local file artifact, got: ${uri}`);
   }
   return resolve2(uri);
 }
@@ -526,6 +560,10 @@ function httpHost(host) {
 }
 function buildModelServerLaunch(args) {
   const options = args.options ?? {};
+  const device = options.device ?? args.config.evaluation.inference.device;
+  if (device !== "cuda") throw new Error("The pinned vLLM serving runtime requires CUDA; CPU serving is not supported.");
+  const contextLength = boundedNumber("contextLength", options.contextLength ?? 16384, 512, 131072, true);
+  const gpuMemoryUtilization = boundedNumber("gpuMemoryUtilization", options.gpuMemoryUtilization ?? 0.8, 0.05, 0.95);
   const host = options.host ?? "127.0.0.1";
   const remote = !isLoopbackHost(host);
   if (remote && !options.allowRemote) {
@@ -561,8 +599,8 @@ function buildModelServerLaunch(args) {
     options.baseModelRevision
   );
   const baseModelRevision = localBaseModelPath ? defaultBaseModelRevision(args.baseModel) : requestedBaseModelRevision;
-  const entrypoint = buildBundledPythonCommand("serve.py");
-  const env = withBundledPythonEnvironment(
+  const entrypoint = buildServingPythonCommand();
+  const env = withServingPythonEnvironment(
     withOfflineHuggingFaceCacheEnvironment({
       ...minimalMachineLearningEnvironment(process.env),
       ...artifactPath ? { TT_MODEL_ARTIFACT: artifactPath } : {},
@@ -570,10 +608,12 @@ function buildModelServerLaunch(args) {
       ...localBaseModelPath ? { TT_MODEL_SOURCE: localBaseModelPath } : {},
       ...baseModelRevision ? { TT_BASE_MODEL_REVISION: baseModelRevision } : {},
       TT_MODEL_NAME: args.modelName,
+      TT_CONTEXT_LENGTH: String(contextLength),
+      TT_GPU_MEMORY_UTILIZATION: String(gpuMemoryUtilization),
       TT_MODEL_LOADER: resolveModelLoader(args.baseModel),
       TT_HOST: host,
       TT_PORT: String(port),
-      TT_DEVICE: options.device ?? args.config.evaluation.inference.device,
+      TT_DEVICE: device,
       TT_MAX_TOKENS: String(maxTokens),
       TT_TEMPERATURE: String(temperature),
       TT_TOP_P: String(topP),
@@ -618,6 +658,7 @@ async function verifyLocalModelServerLaunch(launch) {
   }
 }
 async function serveLocalModel(launch) {
+  if (process.platform !== "linux") throw new Error("Local vLLM serving requires Linux and an NVIDIA CUDA GPU.");
   await verifyLocalModelServerLaunch(launch);
   await new Promise((resolveServer, reject) => {
     const child = spawn2(launch.command, launch.commandArgs, {
@@ -673,12 +714,12 @@ async function serveLocalModel(launch) {
 // src/local-runtime/active-model.ts
 import { randomUUID } from "crypto";
 import { appendFile, chmod, readFile, rename, writeFile as writeFile2 } from "fs/promises";
-import { join as join2 } from "path";
+import { join as join3 } from "path";
 function pointerPath(store) {
-  return join2(store.root, "active-model.json");
+  return join3(store.root, "active-model.json");
 }
 function historyPath(store) {
-  return join2(store.root, "activation-history.jsonl");
+  return join3(store.root, "activation-history.jsonl");
 }
 async function readPointer(store) {
   if (!await hardenExistingLocalStore(store.root)) return null;
@@ -718,7 +759,7 @@ async function assertModelCanActivate(store, modelId) {
     throw new Error(`Model ${model.id} belongs to run ${run.id}, which is not completed.`);
   }
   const report = runReportSchema.parse(
-    JSON.parse(await readFile(join2(model.artifact_dir, "run-report.json"), "utf8"))
+    JSON.parse(await readFile(join3(model.artifact_dir, "run-report.json"), "utf8"))
   );
   if (!report.general_regression) {
     throw new Error(`Model ${model.id} has no general regression evaluation.`);
@@ -775,7 +816,7 @@ async function rollbackActiveModel(store) {
 function packageVersion() {
   try {
     const packageJson = JSON.parse(
-      readFileSync(resolve3(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8")
+      readFileSync2(resolve3(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8")
     );
     return typeof packageJson.version === "string" ? packageJson.version : "unknown";
   } catch {
@@ -830,21 +871,24 @@ var MODEL_SERVE_OPTIONS = [
   CONFIG_OPTION,
   { name: "--host", value: "host", description: "Bind host (localhost by default)" },
   { name: "--port", value: "port", description: "Bind port" },
-  { name: "--device", value: "device", description: "cuda or cpu" },
+  { name: "--device", value: "device", description: "cuda (pinned vLLM runtime)" },
   { name: "--max-tokens", value: "count", description: "Default response token limit" },
   { name: "--temperature", value: "number", description: "Default sampling temperature" },
   { name: "--top-p", value: "number", description: "Default nucleus sampling threshold" },
-  { name: "--max-concurrent-requests", value: "count", description: "Concurrent generation limit" },
+  { name: "--max-concurrent-requests", value: "count", description: "Maximum sequences in an upstream generation batch" },
+  { name: "--context-length", value: "count", description: "Total context token budget (default 16384)" },
+  { name: "--gpu-memory-utilization", value: "fraction", description: "vLLM GPU memory budget fraction (default 0.8)" },
   { name: "--spec", value: "path", description: "Behavior spec whose instructions are enforced" },
   { name: "--no-spec-prompt", description: "Do not enforce the stored behavior-spec prompt" },
   { name: "--allow-remote", description: "Allow a non-loopback bind" },
   { name: "--api-key-env", value: "name", description: "Environment variable containing a bearer token" },
-  { name: "--print-command", description: "Validate and print the launch plan without starting" }
+  { name: "--print-command", description: "Validate and print the launch plan without starting" },
+  { name: "--print-client-config", value: "client", description: "Print Pi models.json; does not start serving" }
 ];
 var COMMAND_DEFINITIONS = {
   info: {
     usage: "tt info",
-    description: "Show the installed TT Local version and runner status.",
+    description: "Show the installed TT local runtime version and runner status.",
     options: [],
     maxPositionals: 0
   },
@@ -1102,7 +1146,7 @@ function readNumberOption(argv, name) {
 async function selectedConfigPath(argv, adjacentTo) {
   const explicitPath = readOption(argv, "--config");
   if (explicitPath) return resolve3(explicitPath);
-  const candidate = join3(adjacentTo ? dirname(resolve3(adjacentTo)) : cwd(), "local-runner.json");
+  const candidate = join4(adjacentTo ? dirname(resolve3(adjacentTo)) : cwd(), "local-runner.json");
   const metadata = await stat(candidate).catch(() => null);
   return metadata?.isFile() ? candidate : void 0;
 }
@@ -1126,7 +1170,7 @@ async function loadCliBehaviorSpec(inputPath, runId) {
     );
   }
   if (input.kind !== "spec") {
-    throw new Error(`TT Local CLI expects a tunedtensor.json behavior spec, not a full run request: ${input.path}`);
+    throw new Error(`TT expects a tunedtensor.json behavior spec, not a full run request: ${input.path}`);
   }
   return input;
 }
@@ -1146,7 +1190,7 @@ function formatEvent(event) {
     const formatted = key === "command" && Array.isArray(value) ? value.join(" ") : shortValue(value);
     return formatted ? `${key}=${formatted}` : null;
   }).filter((value) => Boolean(value)).slice(0, 5).join(" ");
-  return sanitizeLogLine(`[tt-local] ${event.stage}: ${event.message}${detailText ? ` (${detailText})` : ""}`);
+  return sanitizeLogLine(`[tt] ${event.stage}: ${event.message}${detailText ? ` (${detailText})` : ""}`);
 }
 function createConsoleReporter(options) {
   if (options.quiet) return void 0;
@@ -1158,7 +1202,7 @@ function createConsoleReporter(options) {
 `);
     },
     onLog(log) {
-      const line = sanitizeLogLine(`[tt-local] ${log.stage}${log.stream ? ` ${log.stream}` : ""}: ${log.message}`);
+      const line = sanitizeLogLine(`[tt] ${log.stage}${log.stream ? ` ${log.stream}` : ""}: ${log.message}`);
       if (line === lastLogLine) return;
       lastLogLine = line;
       process.stderr.write(`${line}
@@ -1167,7 +1211,7 @@ function createConsoleReporter(options) {
   };
 }
 async function verifyStoredModel(model, config) {
-  const manifestPath = join3(model.artifact_dir, "artifact-manifest.json");
+  const manifestPath = join4(model.artifact_dir, "artifact-manifest.json");
   const integrity = await assertArtifactManifest(manifestPath, {
     requiredPaths: ["stage-metadata.json", "training-report.json"],
     scopeToRequired: true,
@@ -1202,8 +1246,8 @@ async function verifyStoredModel(model, config) {
   };
 }
 async function verifyActivationEvidence(model) {
-  const generalBaseline = join3(model.artifact_dir, "general-baseline-eval.json");
-  const generalCandidate = join3(model.artifact_dir, "general-candidate-eval.json");
+  const generalBaseline = join4(model.artifact_dir, "general-baseline-eval.json");
+  const generalCandidate = join4(model.artifact_dir, "general-candidate-eval.json");
   const [hasBaseline, hasCandidate] = await Promise.all([
     stat(generalBaseline).then((metadata) => metadata.isFile(), () => false),
     stat(generalCandidate).then((metadata) => metadata.isFile(), () => false)
@@ -1213,7 +1257,7 @@ async function verifyActivationEvidence(model) {
       "This model cannot be activated because the run did not include a general-regression suite. Add evaluation.generalRegression to local-runner.json with a held-out dataset, re-run training, then activate. Until then, serve the adapter with `tt serve local-<run-id>`."
     );
   }
-  await assertArtifactManifest(join3(model.artifact_dir, "artifact-manifest.json"), {
+  await assertArtifactManifest(join4(model.artifact_dir, "artifact-manifest.json"), {
     requiredPaths: [
       "run-report.json",
       "general-baseline-eval.json",
@@ -1239,7 +1283,7 @@ async function verifyModelArtifactPath(input, config) {
     if (!metadata) throw new Error(`Model not found and artifact path does not exist: ${input}`);
     let current = metadata.isDirectory() ? inputPath : dirname(inputPath);
     for (let depth = 0; depth < 8; depth += 1) {
-      const candidate = join3(current, "artifact-manifest.json");
+      const candidate = join4(current, "artifact-manifest.json");
       const raw = await readFile2(candidate, "utf8").catch(() => null);
       if (raw) {
         const parsed = JSON.parse(raw);
@@ -1297,7 +1341,7 @@ async function modelSystemPrompt(args) {
     }
     spec = local.data;
   } else {
-    const runRequestPath = join3(args.store.paths.runsDir, args.model.run_id, "request.json");
+    const runRequestPath = join4(args.store.paths.runsDir, args.model.run_id, "request.json");
     let persistedRequest = null;
     try {
       persistedRequest = fineTuneRunRequestSchema.parse(JSON.parse(await readFile2(runRequestPath, "utf8")));
@@ -1313,9 +1357,9 @@ async function modelSystemPrompt(args) {
   }
   const prompt = buildSystemMessage(spec);
   const metadata = JSON.parse(
-    await readFile2(join3(args.model.artifact_dir, "stage-metadata.json"), "utf8")
+    await readFile2(join4(args.model.artifact_dir, "stage-metadata.json"), "utf8")
   );
-  const promptHash = createHash("sha256").update(prompt).digest("hex");
+  const promptHash = createHash2("sha256").update(prompt).digest("hex");
   if (metadata.system_prompt_sha256 !== promptHash) {
     throw new Error(
       "Behavior spec instructions do not match the prompt fingerprint used for this trained model. Pass the original --spec, or use --no-spec-prompt only when this change is intentional."
@@ -1330,6 +1374,47 @@ function modelServeDevice(argv) {
   }
   throw new Error(`--device must be cuda or cpu; got: ${value}`);
 }
+function printServeClientConfig(argv, launch) {
+  const client = readOption(argv, "--print-client-config");
+  if (client === void 0) return false;
+  if (client !== "pi") throw new Error("--print-client-config must be pi.");
+  if (hasFlag(argv, "--print-command")) throw new Error("Use only one of --print-command or --print-client-config.");
+  const apiKeyEnv = readOption(argv, "--api-key-env");
+  if (apiKeyEnv && !/^[A-Za-z_][A-Za-z0-9_]*$/.test(apiKeyEnv)) {
+    throw new Error("--api-key-env must be a plain environment variable name for Pi config.");
+  }
+  const url = new URL(launch.url);
+  if (url.hostname === "0.0.0.0") url.hostname = "127.0.0.1";
+  if (url.hostname === "[::]") url.hostname = "[::1]";
+  url.pathname = "/v1";
+  printJson({
+    providers: {
+      "tt-local": {
+        baseUrl: url.toString(),
+        api: "openai-completions",
+        apiKey: apiKeyEnv ? "${" + apiKeyEnv + "}" : "tt-local",
+        compat: {
+          supportsStore: false,
+          supportsDeveloperRole: false,
+          supportsReasoningEffort: false,
+          supportsUsageInStreaming: true,
+          maxTokensField: "max_tokens"
+        },
+        models: [.../* @__PURE__ */ new Set([launch.modelName, `base:${launch.env.TT_BASE_MODEL}`])].map((id) => ({
+          id,
+          name: id,
+          // This endpoint exports ordinary text/tool calls, not reasoning-effort controls.
+          reasoning: false,
+          input: ["text"],
+          cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+          contextWindow: Number(launch.env.TT_CONTEXT_LENGTH),
+          maxTokens: Number(launch.env.TT_MAX_TOKENS)
+        }))
+      }
+    }
+  });
+  return true;
+}
 async function serveStoredModelFromCli(args) {
   const store = createLocalStore(args.config.storeRoot);
   const model = await store.getModel(args.modelId);
@@ -1345,6 +1430,8 @@ async function serveStoredModelFromCli(args) {
       temperature: readNumberOption(args.argv, "--temperature"),
       topP: readNumberOption(args.argv, "--top-p"),
       maxConcurrentRequests: readNumberOption(args.argv, "--max-concurrent-requests"),
+      contextLength: readNumberOption(args.argv, "--context-length"),
+      gpuMemoryUtilization: readNumberOption(args.argv, "--gpu-memory-utilization"),
       systemPrompt: await modelSystemPrompt({ argv: args.argv, model, store }),
       allowRemote: hasFlag(args.argv, "--allow-remote"),
       apiKeyEnv: readOption(args.argv, "--api-key-env"),
@@ -1358,6 +1445,7 @@ async function serveStoredModelFromCli(args) {
       })()
     }
   });
+  if (printServeClientConfig(args.argv, launch)) return;
   if (hasFlag(args.argv, "--print-command")) {
     printJson({
       ok: true,
@@ -1370,7 +1458,7 @@ async function serveStoredModelFromCli(args) {
     });
     return;
   }
-  process.stderr.write(`[tt-local] verified ${verified.integrity.checked} artifact file(s)
+  process.stderr.write(`[tt] verified ${verified.integrity.checked} artifact file(s)
 `);
   await serveLocalModel(launch);
 }
@@ -1383,6 +1471,8 @@ function serveOptionsFromArgv(argv) {
     temperature: readNumberOption(argv, "--temperature"),
     topP: readNumberOption(argv, "--top-p"),
     maxConcurrentRequests: readNumberOption(argv, "--max-concurrent-requests"),
+    contextLength: readNumberOption(argv, "--context-length"),
+    gpuMemoryUtilization: readNumberOption(argv, "--gpu-memory-utilization"),
     allowRemote: hasFlag(argv, "--allow-remote"),
     apiKeyEnv: readOption(argv, "--api-key-env")
   };
@@ -1421,6 +1511,7 @@ async function serveBaseModelFromCli(args) {
       baseModelRevision: spec?.hyperparameters?.base_model_revision ?? defaultBaseModelRevision(baseModel)
     }
   });
+  if (printServeClientConfig(args.argv, launch)) return;
   if (hasFlag(args.argv, "--print-command")) {
     printJson({
       ok: true,
@@ -1496,7 +1587,7 @@ async function main(argv) {
       } else if (input.kind === "spec") {
         request = input.request;
       } else {
-        throw new Error(`TT Local CLI expects a tunedtensor.json behavior spec, not a full run request: ${input.path}`);
+        throw new Error(`TT expects a tunedtensor.json behavior spec, not a full run request: ${input.path}`);
       }
     }
     const checks = await runDoctor(configSelection.config, request, foundationSpec);
@@ -1583,7 +1674,7 @@ async function main(argv) {
       return;
     }
     if (loaded.kind !== "spec") {
-      throw new Error(`TT Local CLI expects a tunedtensor.json behavior spec, not a full run request: ${loaded.path}`);
+      throw new Error(`TT expects a tunedtensor.json behavior spec, not a full run request: ${loaded.path}`);
     }
     const input = loaded;
     assertLocalRunInputReady(input.request);