@tuned-tensor/cli 0.4.11 → 0.4.12

package/dist/index.js

@@ -1004,17 +1004,374 @@ function registerDatasetsCommands(parent) {
 }
 
 // src/commands/models.ts
-import { existsSync as existsSync3, mkdirSync as mkdirSync2, statSync as statSync3, createWriteStream, unlinkSync } from "fs";
-import { dirname, join as join2 } from "path";
-import { Readable } from "stream";
+import { spawn, execFileSync } from "child_process";
+import { createHash } from "crypto";
+import {
+  existsSync as existsSync4,
+  mkdirSync as mkdirSync2,
+  statSync as statSync3,
+  createWriteStream,
+  unlinkSync,
+  readFileSync as readFileSync6,
+  writeFileSync as writeFileSync3,
+  readdirSync,
+  rmSync
+} from "fs";
+import { homedir as homedir2 } from "os";
+import { dirname, join as join2, resolve as resolve2, basename as basename2 } from "path";
+import { Readable, Transform } from "stream";
 import { pipeline } from "stream/promises";
+
+// src/commands/init.ts
+import { existsSync as existsSync3, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "fs";
+import { resolve } from "path";
+var DEFAULT_SPEC_FILE = "tunedtensor.json";
+var SCAFFOLD = {
+  name: "My Agent",
+  description: "",
+  base_model: "Qwen/Qwen3.5-2B",
+  system_prompt: "You are a helpful assistant.",
+  guidelines: [],
+  constraints: [],
+  examples: [
+    { input: "Hello", output: "Hi! How can I help you today?" }
+  ],
+  eval_cases: []
+};
+function registerInitCommand(parent) {
+  parent.command("init").description("Create a local behaviour spec file").option("-n, --name <name>", "Spec name").option("--model <model>", "Base model ID").option("-f, --file <path>", "Output file path", DEFAULT_SPEC_FILE).action(async (cmdOpts) => {
+    const filePath = resolve(cmdOpts.file);
+    if (existsSync3(filePath)) {
+      printWarning(`${cmdOpts.file} already exists. Use tt eval to validate it or edit it directly.`);
+      return;
+    }
+    const spec = { ...SCAFFOLD };
+    if (cmdOpts.name) spec.name = cmdOpts.name;
+    if (cmdOpts.model) spec.base_model = canonicalizeBaseModel(cmdOpts.model);
+    writeFileSync2(filePath, JSON.stringify(spec, null, 2) + "\n");
+    printSuccess(`Created ${cmdOpts.file}`);
+    console.log("\nNext steps:");
+    console.log("  1. Edit the spec: system_prompt, guidelines, examples");
+    console.log("  2. Validate spec:    tt eval");
+    console.log("  3. Push to remote:   tt push");
+  });
+}
+function loadSpec(filePath) {
+  const resolved = resolve(filePath || DEFAULT_SPEC_FILE);
+  if (!existsSync3(resolved)) {
+    printError(
+      `Spec file not found: ${filePath || DEFAULT_SPEC_FILE}
+Run \`tt init\` to create one.`
+    );
+    process.exit(1);
+  }
+  const raw = JSON.parse(readFileSync5(resolved, "utf-8"));
+  return raw;
+}
+
+// src/commands/models.ts
+var REFERENCE_SERVER_SCRIPT = String.raw`#!/usr/bin/env python3
+import json
+import os
+import time
+import uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+MODEL_PATH = os.environ["TT_MODEL_PATH"]
+MODEL_NAME = os.environ.get("TT_MODEL_NAME", os.path.basename(MODEL_PATH.rstrip("/")) or "tuned-tensor-model")
+SYSTEM_PROMPT = os.environ.get("TT_SYSTEM_PROMPT", "").strip()
+HOST = os.environ.get("TT_HOST", "127.0.0.1")
+PORT = int(os.environ.get("TT_PORT", "8000"))
+DEFAULT_MAX_TOKENS = int(os.environ.get("TT_MAX_TOKENS", "512"))
+DEFAULT_TEMPERATURE = float(os.environ.get("TT_TEMPERATURE", "0.7"))
+TRUST_REMOTE_CODE = os.environ.get("TT_TRUST_REMOTE_CODE", "false").lower() in {"1", "true", "yes", "y"}
+REQUESTED_DEVICE = os.environ.get("TT_DEVICE", "auto").lower()
+
+
+def choose_device(torch):
+    if REQUESTED_DEVICE not in {"auto", "cpu", "cuda", "mps"}:
+        raise RuntimeError("TT_DEVICE must be one of: auto, cpu, cuda, mps")
+    if REQUESTED_DEVICE == "cuda" and not torch.cuda.is_available():
+        raise RuntimeError("TT_DEVICE=cuda requested, but CUDA is not available")
+    if REQUESTED_DEVICE == "mps" and not (
+        hasattr(torch.backends, "mps") and torch.backends.mps.is_available()
+    ):
+        raise RuntimeError("TT_DEVICE=mps requested, but Apple MPS is not available")
+    if REQUESTED_DEVICE != "auto":
+        return REQUESTED_DEVICE
+    if torch.cuda.is_available():
+        return "cuda"
+    if hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+        return "mps"
+    return "cpu"
+
+
+def load_model():
+    try:
+        import torch
+        from transformers import (
+            AutoModelForCausalLM,
+            AutoModelForImageTextToText,
+            AutoProcessor,
+            AutoTokenizer,
+        )
+    except Exception as exc:
+        raise RuntimeError(
+            "Missing Python serving dependencies. Install them with: "
+            "python -m pip install torch transformers accelerate"
+        ) from exc
+
+    try:
+        processor = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=TRUST_REMOTE_CODE)
+    except Exception:
+        processor = AutoProcessor.from_pretrained(MODEL_PATH, trust_remote_code=TRUST_REMOTE_CODE)
+    tokenizer = getattr(processor, "tokenizer", processor)
+    if getattr(tokenizer, "pad_token", None) is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    device = choose_device(torch)
+    kwargs = {
+        "trust_remote_code": TRUST_REMOTE_CODE,
+        "torch_dtype": "auto",
+        "low_cpu_mem_usage": True,
+    }
+    if device == "cuda":
+        kwargs["device_map"] = "auto"
+    elif device == "mps":
+        kwargs["torch_dtype"] = torch.float16
+
+    try:
+        model = AutoModelForCausalLM.from_pretrained(MODEL_PATH, **kwargs)
+    except Exception:
+        model = AutoModelForImageTextToText.from_pretrained(MODEL_PATH, **kwargs)
+    if device != "cuda":
+        model.to(device)
+    model.eval()
+    return torch, tokenizer, model, device
+
+
+torch, tokenizer, model, DEVICE = load_model()
+
+
+def content_to_text(content):
+    if isinstance(content, str):
+        return content
+    return json.dumps(content, ensure_ascii=False)
+
+
+def normalize_messages(raw_messages):
+    messages = []
+    for item in raw_messages:
+        if not isinstance(item, dict):
+            continue
+        role = item.get("role")
+        if role not in {"system", "user", "assistant", "tool"}:
+            continue
+        messages.append({"role": role, "content": content_to_text(item.get("content", ""))})
+
+    if SYSTEM_PROMPT and not any(message["role"] == "system" for message in messages):
+        messages.insert(0, {"role": "system", "content": SYSTEM_PROMPT})
+    return messages
+
+
+def render_prompt(messages):
+    chat_template = getattr(tokenizer, "chat_template", None)
+    if hasattr(tokenizer, "apply_chat_template") and chat_template:
+        return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+
+    rendered = []
+    for message in messages:
+        rendered.append(f"{message['role'].upper()}: {message['content']}")
+    rendered.append("ASSISTANT:")
+    return "\n".join(rendered)
+
+
+def generate_completion(messages, max_tokens, temperature):
+    prompt = render_prompt(messages)
+    inputs = tokenizer(prompt, return_tensors="pt")
+    input_device = getattr(model, "device", None)
+    if input_device is None:
+        input_device = next(model.parameters()).device
+    inputs = {key: value.to(input_device) for key, value in inputs.items()}
+
+    generate_kwargs = {
+        **inputs,
+        "max_new_tokens": max_tokens,
+        "pad_token_id": tokenizer.pad_token_id or tokenizer.eos_token_id,
+    }
+    if temperature > 0:
+        generate_kwargs["do_sample"] = True
+        generate_kwargs["temperature"] = temperature
+    else:
+        generate_kwargs["do_sample"] = False
+
+    with torch.no_grad():
+        output_ids = model.generate(**generate_kwargs)
+
+    prompt_tokens = int(inputs["input_ids"].shape[-1])
+    completion_ids = output_ids[0][prompt_tokens:]
+    content = tokenizer.decode(completion_ids, skip_special_tokens=True).strip()
+    return content, prompt_tokens, int(completion_ids.shape[-1])
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = "TunedTensorReferenceServer/0.1"
+
+    def send_json(self, status, payload):
+        body = json.dumps(payload).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.send_header("Access-Control-Allow-Headers", "authorization, content-type")
+        self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def do_OPTIONS(self):
+        self.send_json(204, {})
+
+    def do_GET(self):
+        if self.path == "/health":
+            self.send_json(200, {
+                "status": "ok",
+                "model": MODEL_NAME,
+                "spec_prompt": bool(SYSTEM_PROMPT),
+                "device": DEVICE,
+            })
+            return
+        self.send_json(404, {"error": {"message": "Not found"}})
+
+    def do_POST(self):
+        if self.path not in {"/v1/chat/completions", "/chat/completions"}:
+            self.send_json(404, {"error": {"message": "Not found"}})
+            return
+
+        try:
+            length = int(self.headers.get("content-length", "0"))
+            body = json.loads(self.rfile.read(length) or b"{}")
+            if body.get("stream"):
+                self.send_json(400, {"error": {"message": "Streaming is not supported by tt models serve yet."}})
+                return
+
+            raw_messages = body.get("messages")
+            if not isinstance(raw_messages, list) or not raw_messages:
+                self.send_json(400, {"error": {"message": "Request must include a non-empty messages array."}})
+                return
+
+            max_tokens = int(body.get("max_tokens") or DEFAULT_MAX_TOKENS)
+            temperature = float(body.get("temperature") if body.get("temperature") is not None else DEFAULT_TEMPERATURE)
+            messages = normalize_messages(raw_messages)
+            content, prompt_tokens, completion_tokens = generate_completion(messages, max_tokens, temperature)
+
+            self.send_json(200, {
+                "id": "chatcmpl-" + uuid.uuid4().hex,
+                "object": "chat.completion",
+                "created": int(time.time()),
+                "model": body.get("model") or MODEL_NAME,
+                "choices": [{
+                    "index": 0,
+                    "message": {"role": "assistant", "content": content},
+                    "finish_reason": "stop",
+                }],
+                "usage": {
+                    "prompt_tokens": prompt_tokens,
+                    "completion_tokens": completion_tokens,
+                    "total_tokens": prompt_tokens + completion_tokens,
+                },
+            })
+        except Exception as exc:
+            self.send_json(500, {"error": {"message": str(exc)}})
+
+    def log_message(self, fmt, *args):
+        print("%s - %s" % (self.address_string(), fmt % args))
+
+
+def main():
+    print(f"Serving {MODEL_NAME} from {MODEL_PATH}")
+    print(f"Listening on http://{HOST}:{PORT}/v1/chat/completions")
+    if SYSTEM_PROMPT:
+        print("Behavior spec prompt: enabled")
+    else:
+        print("Behavior spec prompt: disabled")
+    print(f"Device: {DEVICE}")
+    ThreadingHTTPServer((HOST, PORT), Handler).serve_forever()
+
+
+if __name__ == "__main__":
+    main()
+`;
 function resolveOutputPath(output, filename) {
   if (!output) return filename;
-  if (existsSync3(output) && statSync3(output).isDirectory()) {
+  if (existsSync4(output) && statSync3(output).isDirectory()) {
     return join2(output, filename);
   }
   return output;
 }
+function parseContentLength(value) {
+  if (!value) return null;
+  const parsed = Number(value);
+  return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
+}
+function formatBytes2(bytes) {
+  const units = ["B", "KB", "MB", "GB", "TB"];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit += 1;
+  }
+  const precision = unit === 0 || value >= 10 ? 0 : 1;
+  return `${value.toFixed(precision)} ${units[unit]}`;
+}
+function formatDuration(seconds) {
+  if (!Number.isFinite(seconds) || seconds < 0) return "--";
+  const rounded = Math.max(1, Math.round(seconds));
+  const hours = Math.floor(rounded / 3600);
+  const minutes = Math.floor(rounded % 3600 / 60);
+  const secs = rounded % 60;
+  if (hours > 0) return `${hours}h ${minutes}m`;
+  if (minutes > 0) return `${minutes}m ${secs}s`;
+  return `${secs}s`;
+}
+function chunkLength(chunk) {
+  if (typeof chunk === "string") return Buffer.byteLength(chunk);
+  if (chunk instanceof Uint8Array) return chunk.byteLength;
+  return Buffer.byteLength(String(chunk));
+}
+function createDownloadProgress(totalBytes, enabled) {
+  const stream = process.stderr;
+  const start = Date.now();
+  let lastRender = 0;
+  let rendered = false;
+  function render(downloadedBytes, force = false) {
+    if (!enabled) return;
+    const now = Date.now();
+    if (!force && now - lastRender < 100 && downloadedBytes !== totalBytes) return;
+    lastRender = now;
+    rendered = true;
+    const elapsedSeconds = Math.max((now - start) / 1e3, 1e-3);
+    const rate = downloadedBytes / elapsedSeconds;
+    const rateText = `${formatBytes2(rate)}/s`;
+    let line;
+    if (totalBytes != null && totalBytes > 0) {
+      const ratio = Math.min(downloadedBytes / totalBytes, 1);
+      const barWidth = 24;
+      const filled = Math.round(ratio * barWidth);
+      const bar = "#".repeat(filled) + "-".repeat(barWidth - filled);
+      const etaSeconds = rate > 0 ? (totalBytes - downloadedBytes) / rate : Number.NaN;
+      line = `Downloading [${bar}] ${(ratio * 100).toFixed(1).padStart(5)}% ${formatBytes2(downloadedBytes)}/${formatBytes2(totalBytes)} ${rateText} ETA ${formatDuration(etaSeconds)}`;
+    } else {
+      line = `Downloading ${formatBytes2(downloadedBytes)} ${rateText} elapsed ${formatDuration(elapsedSeconds)}`;
+    }
+    const columns = stream.columns || 120;
+    stream.write(`\r\x1B[K${line.slice(0, Math.max(columns - 1, 0))}`);
+  }
+  function stop() {
+    if (enabled && rendered) stream.write("\n");
+  }
+  return { render, stop };
+}
 async function downloadUrlToFile(url, outputPath) {
   const res = await fetch(url);
   if (!res.ok) {
@@ -1025,17 +1382,225 @@ async function downloadUrlToFile(url, outputPath) {
   }
   mkdirSync2(dirname(outputPath), { recursive: true });
   const file = createWriteStream(outputPath);
+  const contentLength = parseContentLength(res.headers.get("content-length"));
+  const progress = createDownloadProgress(
+    contentLength,
+    !isJsonMode() && process.stderr.isTTY === true
+  );
+  let downloadedBytes = 0;
+  const progressStream = new Transform({
+    transform(chunk, _encoding, callback) {
+      downloadedBytes += chunkLength(chunk);
+      progress.render(downloadedBytes);
+      callback(null, chunk);
+    }
+  });
   try {
-    await pipeline(Readable.fromWeb(res.body), file);
+    progress.render(0, true);
+    await pipeline(Readable.fromWeb(res.body), progressStream, file);
+    progress.render(downloadedBytes, true);
+    progress.stop();
   } catch (err) {
+    progress.stop();
     try {
       unlinkSync(outputPath);
     } catch {
     }
     throw err;
   }
-  const length = res.headers.get("content-length");
-  return length ? Number(length) : null;
+  return contentLength;
+}
+function defaultCacheDir() {
+  const root = process.env.XDG_CACHE_HOME || join2(homedir2(), ".cache");
+  return join2(root, "tuned-tensor", "models");
+}
+function runtimeDir(cacheDir) {
+  return join2(cacheDir, "_runtime");
+}
+function runtimePythonPath(cacheDir) {
+  const dir = runtimeDir(cacheDir);
+  return process.platform === "win32" ? join2(dir, "Scripts", "python.exe") : join2(dir, "bin", "python");
+}
+function resolveServePython(cacheDir, explicitPython) {
+  if (explicitPython) return explicitPython;
+  const managedPython = runtimePythonPath(cacheDir);
+  return existsSync4(managedPython) ? managedPython : "python3";
+}
+function commandExists(command) {
+  try {
+    execFileSync(command, ["--version"], { stdio: "ignore" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+function pythonMinorVersion(command) {
+  try {
+    const out = execFileSync(
+      command,
+      ["-c", "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"],
+      { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }
+    ).trim();
+    const [major, minor] = out.split(".").map((part) => Number(part));
+    if (major !== 3 || !Number.isFinite(minor)) return null;
+    return minor;
+  } catch {
+    return null;
+  }
+}
+function pickRuntimePython(explicitPython) {
+  const candidates = explicitPython ? [explicitPython] : ["python3.12", "python3.11", "python3.10", "python3"];
+  for (const candidate of candidates) {
+    if (!commandExists(candidate)) continue;
+    const minor = pythonMinorVersion(candidate);
+    if (minor != null && minor >= 10 && minor <= 12) return candidate;
+    if (explicitPython) {
+      throw new Error(
+        `${candidate} is Python 3.${minor ?? "?"}. The local serving runtime requires Python 3.10, 3.11, or 3.12 because torch wheels may not be available for newer versions.`
+      );
+    }
+  }
+  throw new Error(
+    "Could not find Python 3.10, 3.11, or 3.12. Install one, then run `tt models setup-runtime --python <path>`."
+  );
+}
+function ensureServingRuntime(python, cacheDir) {
+  const check = `
+import importlib.util
+import json
+required = ["torch", "transformers", "accelerate", "safetensors"]
+missing = [name for name in required if importlib.util.find_spec(name) is None]
+print(json.dumps({"missing": missing}))
+`;
+  let missing;
+  try {
+    const out = execFileSync(python, ["-c", check], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    missing = JSON.parse(out).missing;
+  } catch (err) {
+    const suffix = python === runtimePythonPath(cacheDir) ? " Re-run `tt models setup-runtime` to repair it." : " Run `tt models setup-runtime` to create an isolated serving runtime.";
+    throw new Error(`Python serving runtime check failed for ${python}.${suffix}`);
+  }
+  if (missing.length > 0) {
+    const managedPython = runtimePythonPath(cacheDir);
+    const installHint = python === managedPython ? "Run `tt models setup-runtime --force` to repair it." : "Run `tt models setup-runtime` first, or pass --python <path> to a Python environment with the serving dependencies installed.";
+    throw new Error(
+      `Python serving runtime is missing: ${missing.join(", ")}. ${installHint}`
+    );
+  }
+}
+function setupRuntimeCommands(python, venvDir, deps) {
+  const venvPython = process.platform === "win32" ? join2(venvDir, "Scripts", "python.exe") : join2(venvDir, "bin", "python");
+  return [
+    [python, "-m", "venv", venvDir],
+    [venvPython, "-m", "pip", "install", "--upgrade", "pip"],
+    [venvPython, "-m", "pip", "install", ...deps]
+  ];
+}
+function safeSegment(value) {
+  return value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "model";
+}
+function directoryHasFiles(path) {
+  return existsSync4(path) && statSync3(path).isDirectory() && readdirSync(path).length > 0;
+}
+function extractArchive(archivePath, targetDir, force) {
+  if (directoryHasFiles(targetDir) && !force) return targetDir;
+  if (existsSync4(targetDir)) rmSync(targetDir, { recursive: true, force: true });
+  mkdirSync2(targetDir, { recursive: true });
+  execFileSync("tar", ["-xzf", archivePath, "-C", targetDir], { stdio: "inherit" });
+  return targetDir;
+}
+function localArchiveCacheDir(archivePath, cacheDir) {
+  const resolved = resolve2(archivePath);
+  const digest = createHash("sha256").update(resolved).digest("hex").slice(0, 12);
+  return join2(cacheDir, `local-${safeSegment(basename2(archivePath))}-${digest}`);
+}
+function buildSystemPrompt(spec) {
+  const parts = [];
+  if (spec.system_prompt?.trim()) parts.push(spec.system_prompt.trim());
+  if (Array.isArray(spec.guidelines) && spec.guidelines.length > 0) {
+    parts.push(`Guidelines:
+${spec.guidelines.map((g) => `- ${g}`).join("\n")}`);
+  }
+  if (Array.isArray(spec.constraints) && spec.constraints.length > 0) {
+    parts.push(`Constraints:
+${spec.constraints.map((c) => `- ${c}`).join("\n")}`);
+  }
+  return parts.join("\n\n");
+}
+function findSpecPath(explicitPath, modelPath) {
+  const candidates = explicitPath ? [explicitPath] : [join2(process.cwd(), DEFAULT_SPEC_FILE), join2(modelPath, DEFAULT_SPEC_FILE)];
+  for (const candidate of candidates) {
+    const resolved = resolve2(candidate);
+    if (existsSync4(resolved) && statSync3(resolved).isFile()) return resolved;
+  }
+  return null;
+}
+function loadSystemPromptFromSpec(specPath) {
+  const spec = JSON.parse(readFileSync6(specPath, "utf8"));
+  return buildSystemPrompt(spec);
+}
+function writeReferenceServerScript(cacheDir) {
+  const scriptDir = join2(cacheDir, "_server");
+  mkdirSync2(scriptDir, { recursive: true });
+  const scriptPath = join2(scriptDir, "openai_reference_server.py");
+  writeFileSync3(scriptPath, REFERENCE_SERVER_SCRIPT, "utf8");
+  return scriptPath;
+}
+async function resolveServeTarget(target, opts, cacheDir, forceDownload) {
+  if (existsSync4(target)) {
+    const resolved = resolve2(target);
+    const stats = statSync3(resolved);
+    if (stats.isDirectory()) {
+      return {
+        modelPath: resolved,
+        modelName: basename2(resolved),
+        source: "local-directory"
+      };
+    }
+    if (stats.isFile() && (resolved.endsWith(".tar.gz") || resolved.endsWith(".tgz"))) {
+      const extractDir2 = join2(localArchiveCacheDir(resolved, cacheDir), "model");
+      return {
+        modelPath: extractArchive(resolved, extractDir2, forceDownload),
+        modelName: basename2(resolved).replace(/\.t(ar\.)?gz$/, ""),
+        source: "local-archive"
+      };
+    }
+    throw new Error(`Unsupported model path: ${target}. Use a model directory or .tar.gz artifact.`);
+  }
+  const fullId = await resolveModelId(target, opts);
+  const { data: model } = await get(`/models/${fullId}`, void 0, opts);
+  const { data } = await get(`/models/${fullId}/download`, void 0, opts);
+  const modelCacheDir = join2(cacheDir, fullId);
+  const archivePath = join2(modelCacheDir, data.filename);
+  const extractDir = join2(modelCacheDir, "model");
+  if (!existsSync4(archivePath) || forceDownload) {
+    await downloadUrlToFile(data.url, archivePath);
+  }
+  extractArchive(archivePath, extractDir, forceDownload);
+  return {
+    modelPath: extractDir,
+    modelName: model.name || fullId,
+    source: "downloaded-model",
+    modelId: fullId
+  };
+}
+function quoteCommandPart(part) {
+  if (/^[a-zA-Z0-9_./:=@-]+$/.test(part)) return part;
+  return `'${part.replace(/'/g, "'\\''")}'`;
+}
+function printServeCommand(command, args, env) {
+  const envKeys = Object.keys(env).sort();
+  const commandLine = [command, ...args].map(quoteCommandPart).join(" ");
+  if (isJsonMode()) {
+    return printJson({ command, args, env_keys: envKeys, command_line: commandLine });
+  }
+  printDetail([
+    ["Command", commandLine],
+    ["Environment", envKeys.join(", ")]
+  ]);
 }
 function registerModelsCommands(parent) {
   const models = parent.command("models").description("Manage fine-tuned models");
@@ -1095,7 +1660,7 @@ function registerModelsCommands(parent) {
       opts
     );
     const outputPath = resolveOutputPath(cmdOpts.output, data.filename);
-    if (existsSync3(outputPath) && !cmdOpts.force) {
+    if (existsSync4(outputPath) && !cmdOpts.force) {
       throw new Error(`Output file already exists: ${outputPath}. Use --force to overwrite.`);
     }
     const bytes = await downloadUrlToFile(data.url, outputPath);
@@ -1111,6 +1676,113 @@ function registerModelsCommands(parent) {
     const size = bytes == null ? "" : ` (${bytes} bytes)`;
     printSuccess(`Model downloaded to ${outputPath}${size}`);
   });
+  models.command("setup-runtime").description("Install an isolated Python runtime for local model serving").option("--python <path>", "Python 3.10-3.12 executable to create the runtime").option("--cache-dir <path>", "Cache directory for the managed runtime").option("-f, --force", "Recreate the runtime if it already exists").option("--print-command", "Print the setup commands without running them").action(async (cmdOpts) => {
+    const cacheDir = resolve2(cmdOpts.cacheDir || defaultCacheDir());
+    const venvDir = runtimeDir(cacheDir);
+    const venvPython = runtimePythonPath(cacheDir);
+    const python = pickRuntimePython(cmdOpts.python);
+    const deps = ["torch", "transformers", "accelerate", "safetensors"];
+    const commands = setupRuntimeCommands(python, venvDir, deps);
+    if (cmdOpts.printCommand) {
+      const commandLines = commands.map((cmd) => cmd.map(quoteCommandPart).join(" "));
+      if (isJsonMode()) {
+        return printJson({
+          python,
+          runtime_dir: venvDir,
+          runtime_python: venvPython,
+          commands: commandLines
+        });
+      }
+      printDetail([
+        ["Python", python],
+        ["Runtime", venvDir],
+        ["Commands", commandLines.join("\n")]
+      ]);
+      return;
+    }
+    if (existsSync4(venvDir) && cmdOpts.force) {
+      rmSync(venvDir, { recursive: true, force: true });
+    }
+    if (!existsSync4(venvPython)) {
+      mkdirSync2(cacheDir, { recursive: true });
+      execFileSync(commands[0][0], commands[0].slice(1), { stdio: "inherit" });
+    }
+    execFileSync(commands[1][0], commands[1].slice(1), { stdio: "inherit" });
+    execFileSync(commands[2][0], commands[2].slice(1), { stdio: "inherit" });
+    ensureServingRuntime(venvPython, cacheDir);
+    if (isJsonMode()) {
+      return printJson({
+        runtime_dir: venvDir,
+        runtime_python: venvPython,
+        installed: true,
+        dependencies: deps
+      });
+    }
+    printSuccess(`Serving runtime ready at ${venvDir}`);
+    console.log(`Use it with: tt models serve <model-id> --spec ${DEFAULT_SPEC_FILE}`);
+  });
+  models.command("serve").description("Serve a downloaded model with an OpenAI-compatible local API").argument("<target>", "Model ID/prefix, model directory, or .tar.gz artifact").option("--spec <path>", "Behavior spec JSON to apply as the default system prompt").option("--no-spec-prompt", "Do not auto-apply a behavior spec system prompt").option("--host <host>", "Host to bind", "127.0.0.1").option("--port <port>", "Port to bind", "8000").option("--python <path>", "Python executable").option("--cache-dir <path>", "Cache directory for downloaded/extracted models").option("--device <device>", "Inference device: auto, cpu, cuda, or mps", "auto").option("--force-download", "Re-download and re-extract model artifacts").option("--max-tokens <n>", "Default max completion tokens", "512").option("--temperature <n>", "Default sampling temperature", "0.7").option("--trust-remote-code", "Pass trust_remote_code=True to transformers").option("--print-command", "Print the underlying Python command without starting it").action(async (target, cmdOpts) => {
+    const opts = parent.opts();
+    const cacheDir = resolve2(cmdOpts.cacheDir || defaultCacheDir());
+    mkdirSync2(cacheDir, { recursive: true });
+    const python = resolveServePython(cacheDir, cmdOpts.python);
+    if (!cmdOpts.printCommand) ensureServingRuntime(python, cacheDir);
+    const serveTarget = await resolveServeTarget(
+      target,
+      opts,
+      cacheDir,
+      Boolean(cmdOpts.forceDownload)
+    );
+    const serverScript = writeReferenceServerScript(cacheDir);
+    let specPath = null;
+    let systemPrompt = "";
+    if (cmdOpts.specPrompt !== false) {
+      specPath = findSpecPath(cmdOpts.spec, serveTarget.modelPath);
+      if (cmdOpts.spec && !specPath) {
+        throw new Error(`Spec file not found: ${cmdOpts.spec}`);
+      }
+      if (specPath) {
+        systemPrompt = loadSystemPromptFromSpec(specPath);
+      }
+    }
+    const env = {
+      TT_MODEL_PATH: serveTarget.modelPath,
+      TT_MODEL_NAME: serveTarget.modelName,
+      TT_HOST: cmdOpts.host,
+      TT_PORT: String(cmdOpts.port),
+      TT_MAX_TOKENS: String(cmdOpts.maxTokens),
+      TT_TEMPERATURE: String(cmdOpts.temperature),
+      TT_TRUST_REMOTE_CODE: cmdOpts.trustRemoteCode ? "true" : "false",
+      TT_DEVICE: String(cmdOpts.device)
+    };
+    if (systemPrompt) env.TT_SYSTEM_PROMPT = systemPrompt;
+    const args = [serverScript];
+    if (cmdOpts.printCommand) return printServeCommand(python, args, env);
+    if (!isJsonMode()) {
+      printSuccess(`Serving ${serveTarget.modelName} from ${serveTarget.modelPath}`);
+      if (systemPrompt && specPath) {
+        printSuccess(`Behavior spec prompt loaded from ${specPath}`);
+      } else if (cmdOpts.specPrompt !== false) {
+        printWarning(
+          `No ${DEFAULT_SPEC_FILE} found. Pass --spec <path> to preserve the behavior prompt.`
+        );
+      }
+      console.log(`OpenAI-compatible endpoint: http://${cmdOpts.host}:${cmdOpts.port}/v1/chat/completions`);
+      console.log(`Health check: http://${cmdOpts.host}:${cmdOpts.port}/health`);
+      console.log(`Python runtime: ${python}`);
+    }
+    await new Promise((resolvePromise, reject) => {
+      const child = spawn(python, args, {
+        stdio: "inherit",
+        env: { ...process.env, ...env }
+      });
+      child.on("error", reject);
+      child.on("exit", (code, signal) => {
+        if (code === 0) return resolvePromise();
+        reject(new Error(`Model server exited with ${signal ?? `code ${code}`}`));
+      });
+    });
+  });
   models.command("delete").description("Delete a model").argument("<id>", "Model ID (full UUID or 4+ char prefix)").action(async (id) => {
     const opts = parent.opts();
     const fullId = await resolveModelId(id, opts);
@@ -1274,53 +1946,6 @@ function registerTopupCommands(parent) {
   });
 }
 
-// src/commands/init.ts
-import { existsSync as existsSync4, readFileSync as readFileSync5, writeFileSync as writeFileSync2 } from "fs";
-import { resolve } from "path";
-var DEFAULT_SPEC_FILE = "tunedtensor.json";
-var SCAFFOLD = {
-  name: "My Agent",
-  description: "",
-  base_model: "Qwen/Qwen3.5-2B",
-  system_prompt: "You are a helpful assistant.",
-  guidelines: [],
-  constraints: [],
-  examples: [
-    { input: "Hello", output: "Hi! How can I help you today?" }
-  ],
-  eval_cases: []
-};
-function registerInitCommand(parent) {
-  parent.command("init").description("Create a local behaviour spec file").option("-n, --name <name>", "Spec name").option("--model <model>", "Base model ID").option("-f, --file <path>", "Output file path", DEFAULT_SPEC_FILE).action(async (cmdOpts) => {
-    const filePath = resolve(cmdOpts.file);
-    if (existsSync4(filePath)) {
-      printWarning(`${cmdOpts.file} already exists. Use tt eval to validate it or edit it directly.`);
-      return;
-    }
-    const spec = { ...SCAFFOLD };
-    if (cmdOpts.name) spec.name = cmdOpts.name;
-    if (cmdOpts.model) spec.base_model = canonicalizeBaseModel(cmdOpts.model);
-    writeFileSync2(filePath, JSON.stringify(spec, null, 2) + "\n");
-    printSuccess(`Created ${cmdOpts.file}`);
-    console.log("\nNext steps:");
-    console.log("  1. Edit the spec: system_prompt, guidelines, examples");
-    console.log("  2. Validate spec:    tt eval");
-    console.log("  3. Push to remote:   tt push");
-  });
-}
-function loadSpec(filePath) {
-  const resolved = resolve(filePath || DEFAULT_SPEC_FILE);
-  if (!existsSync4(resolved)) {
-    printError(
-      `Spec file not found: ${filePath || DEFAULT_SPEC_FILE}
-Run \`tt init\` to create one.`
-    );
-    process.exit(1);
-  }
-  const raw = JSON.parse(readFileSync5(resolved, "utf-8"));
-  return raw;
-}
-
 // src/commands/eval.ts
 import chalk5 from "chalk";
 
@@ -1432,12 +2057,12 @@ function printValidation(checks) {
 }
 
 // src/commands/push.ts
-import { readFileSync as readFileSync6, writeFileSync as writeFileSync3 } from "fs";
-import { resolve as resolve2 } from "path";
+import { readFileSync as readFileSync7, writeFileSync as writeFileSync4 } from "fs";
+import { resolve as resolve3 } from "path";
 function registerPushCommand(parent) {
   parent.command("push").description("Push local spec to the Tuned Tensor API").option("-f, --file <path>", "Spec file path", DEFAULT_SPEC_FILE).action(async (cmdOpts) => {
     const opts = parent.opts();
-    const filePath = resolve2(cmdOpts.file);
+    const filePath = resolve3(cmdOpts.file);
     const spec = loadSpec(cmdOpts.file);
     const { id, eval_cases, ...rawBody } = spec;
     const body = canonicalizeSpecBaseModel(rawBody);
@@ -1448,9 +2073,9 @@ function registerPushCommand(parent) {
     } else {
       const res = await post("/behavior-specs", body, opts);
       data = res.data;
-      const raw = JSON.parse(readFileSync6(filePath, "utf-8"));
+      const raw = JSON.parse(readFileSync7(filePath, "utf-8"));
       raw.id = data.id;
-      writeFileSync3(filePath, JSON.stringify(raw, null, 2) + "\n");
+      writeFileSync4(filePath, JSON.stringify(raw, null, 2) + "\n");
     }
     if (isJsonMode()) return printJson(data);
     printSuccess(
@@ -1461,7 +2086,7 @@ function registerPushCommand(parent) {
 
 // src/index.ts
 var program = new Command();
-program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.11").option("-k, --api-key <key>", "API key (overrides stored key)").option(
+program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.12").option("-k, --api-key <key>", "API key (overrides stored key)").option(
   "-u, --base-url <url>",
   "API base URL (default: https://tunedtensor.com)"
 ).option("--json", "Output raw JSON").option("--no-color", "Disable colors").hook("preAction", (_thisCommand, actionCommand) => {