@tuned-tensor/cli 0.4.13 → 0.4.15

package/dist/index.js

@@ -595,6 +595,13 @@ function formatEvalScore(run) {
   const score = getEvalScore(run);
   return score == null ? void 0 : (score * 100).toFixed(1) + "%";
 }
+function formatPercent(rate) {
+  return rate == null ? void 0 : (rate * 100).toFixed(1) + "%";
+}
+function formatCountRate(count, total, rate) {
+  if (count == null || total == null || rate == null) return void 0;
+  return `${formatPercent(rate)} (${count}/${total})`;
+}
 function formatMinutes(minutes) {
   if (minutes == null) return void 0;
   if (minutes < 60) return `${Math.max(1, Math.round(minutes))}m`;
@@ -632,6 +639,78 @@ function printDiagnostics(diagnostics) {
       console.log(`  - ${insight}`);
     }
   }
+  printEvalOutputDiagnostics(diagnostics.output_diagnostics);
+}
+function printEvalOutputDiagnostics(diagnostics) {
+  if (!diagnostics) return;
+  const candidate = diagnostics.candidate;
+  const baseline = diagnostics.baseline;
+  console.log("\nOutput Diagnostics:");
+  printDetail([
+    ["Tuned Valid JSON", formatCountRate(candidate.valid_json_count, candidate.total, candidate.valid_json_rate)],
+    ["Tuned Strict JSON", formatCountRate(candidate.strict_json_count, candidate.total, candidate.strict_json_rate)],
+    [
+      "Tuned Schema Keys",
+      formatCountRate(candidate.expected_schema_keys_count, candidate.total, candidate.expected_schema_keys_rate)
+    ],
+    [
+      "Tuned Non-JSON Prefix",
+      formatCountRate(candidate.non_json_prefix_count, candidate.total, candidate.non_json_prefix_rate)
+    ],
+    [
+      "Tuned Reasoning Prefix",
+      formatCountRate(
+        candidate.visible_reasoning_prefix_count,
+        candidate.total,
+        candidate.visible_reasoning_prefix_rate
+      )
+    ],
+    ["Tuned Avg Output", candidate.avg_output_chars == null ? void 0 : `${candidate.avg_output_chars} chars`],
+    ["Base Valid JSON", formatCountRate(baseline.valid_json_count, baseline.total, baseline.valid_json_rate)],
+    [
+      "Base Reasoning Prefix",
+      formatCountRate(
+        baseline.visible_reasoning_prefix_count,
+        baseline.total,
+        baseline.visible_reasoning_prefix_rate
+      )
+    ]
+  ]);
+  if (diagnostics.test) {
+    console.log("\nTest Output Diagnostics:");
+    printDetail([
+      [
+        "Tuned Valid JSON",
+        formatCountRate(
+          diagnostics.test.candidate.valid_json_count,
+          diagnostics.test.candidate.total,
+          diagnostics.test.candidate.valid_json_rate
+        )
+      ],
+      [
+        "Tuned Strict JSON",
+        formatCountRate(
+          diagnostics.test.candidate.strict_json_count,
+          diagnostics.test.candidate.total,
+          diagnostics.test.candidate.strict_json_rate
+        )
+      ],
+      [
+        "Tuned Reasoning Prefix",
+        formatCountRate(
+          diagnostics.test.candidate.visible_reasoning_prefix_count,
+          diagnostics.test.candidate.total,
+          diagnostics.test.candidate.visible_reasoning_prefix_rate
+        )
+      ]
+    ]);
+  }
+  if (diagnostics.insights?.length) {
+    console.log("\nEvaluation Insights:");
+    for (const insight of diagnostics.insights) {
+      console.log(`  - ${insight}`);
+    }
+  }
 }
 function registerRunsCommands(parent) {
   const runs = parent.command("runs").description("Manage runs");
@@ -687,6 +766,7 @@ function registerRunsCommands(parent) {
         console.log(`  ${k}: ${v}`);
       }
     }
+    printEvalOutputDiagnostics(data.eval_summary?.output_diagnostics);
     if (data._events?.length) {
       console.log("\nLatest Updates:");
       for (const event of data._events.slice(-5)) {
@@ -1009,11 +1089,11 @@ function registerDatasetsCommands(parent) {
 }
 
 // src/commands/models.ts
-import { spawn, execFileSync } from "child_process";
+import { spawn as spawn2, execFileSync } from "child_process";
 import { createHash } from "crypto";
 import {
   existsSync as existsSync4,
-  mkdirSync as mkdirSync2,
+  mkdirSync as mkdirSync3,
   statSync as statSync3,
   createWriteStream,
   unlinkSync,
@@ -1023,7 +1103,7 @@ import {
   rmSync
 } from "fs";
 import { homedir as homedir2 } from "os";
-import { dirname, join as join2, resolve as resolve2, basename as basename2 } from "path";
+import { dirname as dirname2, join as join2, resolve as resolve2, basename as basename2 } from "path";
 import { Readable, Transform } from "stream";
 import { pipeline } from "stream/promises";
 
@@ -1074,6 +1154,444 @@ Run \`tt init\` to create one.`
   return raw;
 }
 
+// src/serve-manager.ts
+import { spawn } from "child_process";
+import { randomUUID } from "crypto";
+import { appendFileSync, mkdirSync as mkdirSync2 } from "fs";
+import { createServer } from "http";
+import { createServer as createNetServer } from "net";
+import { dirname } from "path";
+var COMPLETION_ROUTES = /* @__PURE__ */ new Set(["/v1/chat/completions", "/chat/completions"]);
+function parsePositiveInteger(value, fallback) {
+  return Number.isInteger(value) && value >= 0 ? value : fallback;
+}
+function sleep(ms) {
+  return new Promise((resolve4) => setTimeout(resolve4, ms));
+}
+function readRequestBody(req) {
+  return new Promise((resolve4, reject) => {
+    const chunks = [];
+    req.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
+    req.on("end", () => resolve4(Buffer.concat(chunks)));
+    req.on("error", reject);
+  });
+}
+function sendJson(res, status, payload) {
+  const body = Buffer.from(JSON.stringify(payload), "utf8");
+  res.writeHead(status, {
+    "content-type": "application/json",
+    "content-length": String(body.byteLength),
+    "access-control-allow-origin": "*",
+    "access-control-allow-headers": "authorization, content-type",
+    "access-control-allow-methods": "GET, POST, OPTIONS"
+  });
+  res.end(body);
+}
+function copyResponseHeaders(headers) {
+  const copied = {
+    "access-control-allow-origin": "*",
+    "access-control-allow-headers": "authorization, content-type",
+    "access-control-allow-methods": "GET, POST, OPTIONS"
+  };
+  const contentType = headers.get("content-type");
+  if (contentType) copied["content-type"] = contentType;
+  return copied;
+}
+function tryJsonParse(value) {
+  try {
+    return JSON.parse(value);
+  } catch {
+    return null;
+  }
+}
+function readPath(value, path) {
+  if (!path) return null;
+  let current = value;
+  for (const part of path.split(".")) {
+    if (!part) return null;
+    if (!current || typeof current !== "object" || !(part in current)) return null;
+    current = current[part];
+  }
+  return current ?? null;
+}
+function extractAssistantContent(response) {
+  if (!response || typeof response !== "object") return null;
+  const choices = response.choices;
+  if (!Array.isArray(choices) || choices.length === 0) return null;
+  const first = choices[0];
+  if (!first || typeof first !== "object") return null;
+  const message = first.message;
+  if (!message || typeof message !== "object") return null;
+  const content = message.content;
+  return typeof content === "string" ? content : null;
+}
+function usageNumber(response, key) {
+  if (!response || typeof response !== "object") return null;
+  const usage = response.usage;
+  if (!usage || typeof usage !== "object") return null;
+  const value = usage[key];
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+function errorSummary(response, fallback) {
+  if (!response || typeof response !== "object") return fallback;
+  const error = response.error;
+  if (!error || typeof error !== "object") return fallback;
+  const message = error.message;
+  return typeof message === "string" ? message : fallback;
+}
+async function allocatePort(host) {
+  return new Promise((resolve4, reject) => {
+    const server = createNetServer();
+    server.on("error", reject);
+    server.listen(0, host, () => {
+      const address = server.address();
+      if (!address || typeof address === "string") {
+        server.close(() => reject(new Error("Could not allocate an internal port")));
+        return;
+      }
+      const port = address.port;
+      server.close(() => resolve4(port));
+    });
+  });
+}
+var ManagedModelServer = class {
+  constructor(options) {
+    this.options = options;
+    this.spawnModelServer = options.spawnModelServer ?? spawn;
+    this.idleTimeoutMs = parsePositiveInteger(options.idleTimeoutSeconds, 300) * 1e3;
+    this.restartAfterRequests = parsePositiveInteger(options.restartAfterRequests, 100);
+    this.logRecordOverride = options.logRecord;
+  }
+  spawnModelServer;
+  idleTimeoutMs;
+  restartAfterRequests;
+  logRecordOverride;
+  server = null;
+  child = null;
+  internalPort = null;
+  queue = Promise.resolve();
+  idleTimer = null;
+  activeRequests = 0;
+  queuedRequests = 0;
+  requestsSinceStart = 0;
+  restartCount = 0;
+  stoppingChild = false;
+  get isModelRunning() {
+    return this.child != null;
+  }
+  get restarts() {
+    return this.restartCount;
+  }
+  async start() {
+    if (this.server) return;
+    this.internalPort = await allocatePort("127.0.0.1");
+    this.server = createServer((req, res) => {
+      this.handle(req, res).catch((err) => {
+        sendJson(res, 500, { error: { message: err instanceof Error ? err.message : String(err) } });
+      });
+    });
+    await new Promise((resolve4, reject) => {
+      this.server?.once("error", reject);
+      this.server?.listen(this.options.publicPort, this.options.publicHost, () => {
+        this.server?.off("error", reject);
+        resolve4();
+      });
+    });
+  }
+  async stop() {
+    this.clearIdleTimer();
+    await this.stopChild();
+    if (!this.server) return;
+    const server = this.server;
+    this.server = null;
+    await new Promise((resolve4, reject) => {
+      server.close((err) => err ? reject(err) : resolve4());
+    });
+  }
+  async run() {
+    await this.start();
+    await new Promise((resolve4, reject) => {
+      this.server?.once("error", reject);
+      this.server?.once("close", resolve4);
+    });
+  }
+  async handle(req, res) {
+    const route = new URL(req.url || "/", "http://localhost").pathname;
+    if (req.method === "OPTIONS") {
+      sendJson(res, 204, {});
+      return;
+    }
+    if (req.method === "GET" && route === "/health") {
+      await this.handleHealth(res);
+      return;
+    }
+    if (req.method === "POST" && COMPLETION_ROUTES.has(route)) {
+      const receivedAt = Date.now();
+      const body = await readRequestBody(req);
+      this.queuedRequests += 1;
+      const work = this.queue.then(() => this.processGeneration(route, body, res, receivedAt));
+      this.queue = work.catch(() => void 0);
+      try {
+        await work;
+      } finally {
+        this.queuedRequests -= 1;
+        this.scheduleIdleStop();
+      }
+      return;
+    }
+    sendJson(res, 404, { error: { message: "Not found" } });
+  }
+  async handleHealth(res) {
+    if (!this.child) {
+      sendJson(res, 200, {
+        status: "ok",
+        wrapper: "ok",
+        model_status: "cold",
+        restart_count: this.restartCount
+      });
+      return;
+    }
+    if (this.activeRequests > 0 || this.queuedRequests > 0) {
+      sendJson(res, 200, {
+        status: "ok",
+        wrapper: "ok",
+        model_status: "busy",
+        restart_count: this.restartCount
+      });
+      return;
+    }
+    const health = await this.checkHealth();
+    if (health.ok) {
+      sendJson(res, 200, {
+        status: "ok",
+        wrapper: "ok",
+        model_status: "warm",
+        restart_count: this.restartCount,
+        model: health.body
+      });
+      return;
+    }
+    await this.restartChild();
+    sendJson(res, 200, {
+      status: "ok",
+      wrapper: "ok",
+      model_status: "restarted",
+      restart_count: this.restartCount
+    });
+  }
+  async processGeneration(route, body, res, receivedAt) {
+    const requestId = `ttreq-${randomUUID()}`;
+    const startedAt = Date.now();
+    const queuedMs = startedAt - receivedAt;
+    this.activeRequests += 1;
+    this.clearIdleTimer();
+    let status = 500;
+    let parsed = null;
+    let failure = null;
+    try {
+      await this.ensureModelReady();
+      const forwarded = await this.forwardGeneration(route, body);
+      status = forwarded.status;
+      parsed = forwarded.parsed;
+      res.writeHead(status, {
+        ...copyResponseHeaders(forwarded.headers),
+        "content-length": String(Buffer.byteLength(forwarded.body))
+      });
+      res.end(forwarded.body);
+      this.requestsSinceStart += 1;
+      failure = errorSummary(parsed, null);
+    } catch (err) {
+      failure = err instanceof Error ? err.message : String(err);
+      status = 502;
+      sendJson(res, status, { error: { message: failure } });
+    } finally {
+      const finishedAt = Date.now();
+      this.activeRequests -= 1;
+      this.writeLog({
+        timestamp: new Date(finishedAt).toISOString(),
+        request_id: requestId,
+        route,
+        model_target: {
+          name: this.options.modelName,
+          path: this.options.modelPath
+        },
+        request_bytes: body.byteLength,
+        response_status: status,
+        latency_ms: finishedAt - receivedAt,
+        queued_ms: queuedMs,
+        prompt_tokens: usageNumber(parsed, "prompt_tokens"),
+        completion_tokens: usageNumber(parsed, "completion_tokens"),
+        total_tokens: usageNumber(parsed, "total_tokens"),
+        schema_validity: status < 400 ? true : status === 422 ? false : null,
+        gate_field: this.options.gateField,
+        gate_result: this.extractGateResult(parsed),
+        restart_count: this.restartCount,
+        error_summary: failure
+      });
+    }
+  }
+  async ensureModelReady() {
+    if (this.child && this.restartAfterRequests > 0 && this.requestsSinceStart >= this.restartAfterRequests) {
+      await this.restartChild();
+      return;
+    }
+    if (this.child) {
+      const health = await this.checkHealth();
+      if (health.ok) return;
+      await this.restartChild();
+      return;
+    }
+    await this.startChild();
+  }
+  async startChild() {
+    if (this.internalPort == null) {
+      this.internalPort = await allocatePort("127.0.0.1");
+    }
+    const env = {
+      ...process.env,
+      ...this.options.env,
+      TT_HOST: "127.0.0.1",
+      TT_PORT: String(this.internalPort)
+    };
+    this.child = this.spawnModelServer(this.options.python, this.options.args, {
+      stdio: "inherit",
+      env
+    });
+    this.child.once("exit", () => {
+      if (!this.stoppingChild) {
+        this.child = null;
+        this.requestsSinceStart = 0;
+      }
+    });
+    try {
+      await this.waitForHealth();
+      this.scheduleIdleStop();
+    } catch (err) {
+      await this.stopChild();
+      throw err;
+    }
+  }
+  async restartChild() {
+    await this.stopChild();
+    this.restartCount += 1;
+    this.requestsSinceStart = 0;
+    await this.startChild();
+  }
+  async stopChild() {
+    if (!this.child) return;
+    const child = this.child;
+    this.child = null;
+    this.requestsSinceStart = 0;
+    this.stoppingChild = true;
+    try {
+      child.kill("SIGTERM");
+      await Promise.race([
+        new Promise((resolve4) => child.once("exit", () => resolve4())),
+        sleep(5e3).then(() => {
+          if (child.exitCode == null && child.signalCode == null) child.kill("SIGKILL");
+        })
+      ]);
+    } finally {
+      this.stoppingChild = false;
+    }
+  }
+  async checkHealth() {
+    if (this.internalPort == null) return { ok: false, body: null };
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 2e3);
+    try {
+      const response = await fetch(`http://127.0.0.1:${this.internalPort}/health`, {
+        signal: controller.signal
+      });
+      const text = await response.text();
+      return { ok: response.ok, body: tryJsonParse(text) };
+    } catch {
+      return { ok: false, body: null };
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+  async waitForHealth() {
+    const deadline = Date.now() + 12e4;
+    while (Date.now() < deadline) {
+      const health = await this.checkHealth();
+      if (health.ok) return;
+      if (!this.child) break;
+      await sleep(50);
+    }
+    throw new Error("Model server did not become healthy within 120 seconds");
+  }
+  async forwardGeneration(route, body) {
+    if (this.internalPort == null) throw new Error("Internal server port is not allocated");
+    const response = await fetch(`http://127.0.0.1:${this.internalPort}${route}`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body
+    });
+    const text = await response.text();
+    return {
+      status: response.status,
+      headers: response.headers,
+      body: text,
+      parsed: tryJsonParse(text)
+    };
+  }
+  extractGateResult(response) {
+    const content = extractAssistantContent(response);
+    if (!content) return null;
+    const parsedContent = tryJsonParse(content);
+    return readPath(parsedContent, this.options.gateField);
+  }
+  scheduleIdleStop() {
+    this.clearIdleTimer();
+    if (!this.child || this.activeRequests > 0 || this.queuedRequests > 0) return;
+    this.idleTimer = setTimeout(() => {
+      this.stopChild().catch((err) => {
+        this.writeDiagnosticError(err);
+      });
+    }, this.idleTimeoutMs);
+  }
+  clearIdleTimer() {
+    if (!this.idleTimer) return;
+    clearTimeout(this.idleTimer);
+    this.idleTimer = null;
+  }
+  writeLog(record) {
+    if (this.logRecordOverride) {
+      this.logRecordOverride(record);
+      return;
+    }
+    const line = `${JSON.stringify(record)}
+`;
+    if (this.options.logFile) {
+      mkdirSync2(dirname(this.options.logFile), { recursive: true });
+      appendFileSync(this.options.logFile, line, "utf8");
+      return;
+    }
+    process.stderr.write(line);
+  }
+  writeDiagnosticError(err) {
+    const message = err instanceof Error ? err.message : String(err);
+    process.stderr.write(`${JSON.stringify({ timestamp: (/* @__PURE__ */ new Date()).toISOString(), error: message })}
+`);
+  }
+};
+async function startManagedModelServer(options) {
+  const server = new ManagedModelServer(options);
+  const stop = () => {
+    server.stop().catch(() => void 0);
+  };
+  process.once("SIGINT", stop);
+  process.once("SIGTERM", stop);
+  try {
+    await server.run();
+  } finally {
+    process.off("SIGINT", stop);
+    process.off("SIGTERM", stop);
+    await server.stop();
+  }
+}
+
 // src/commands/models.ts
 var REFERENCE_SERVER_SCRIPT = String.raw`#!/usr/bin/env python3
 import json
@@ -1091,6 +1609,10 @@ DEFAULT_MAX_TOKENS = int(os.environ.get("TT_MAX_TOKENS", "512"))
 DEFAULT_TEMPERATURE = float(os.environ.get("TT_TEMPERATURE", "0.7"))
 TRUST_REMOTE_CODE = os.environ.get("TT_TRUST_REMOTE_CODE", "false").lower() in {"1", "true", "yes", "y"}
 REQUESTED_DEVICE = os.environ.get("TT_DEVICE", "auto").lower()
+JSON_REPAIR_ATTEMPTS = int(os.environ.get("TT_JSON_REPAIR_ATTEMPTS", "1"))
+DEFAULT_JSON_SCHEMA = None
+if os.environ.get("TT_JSON_SCHEMA"):
+    DEFAULT_JSON_SCHEMA = json.loads(os.environ["TT_JSON_SCHEMA"])
 
 
 def choose_device(torch):
@@ -1179,6 +1701,50 @@ def normalize_messages(raw_messages):
     return messages
 
 
+def response_json_contract(body):
+    response_format = body.get("response_format")
+    schema = DEFAULT_JSON_SCHEMA
+    mode = "json_schema" if schema else "text"
+
+    if isinstance(response_format, dict):
+        format_type = response_format.get("type")
+        if format_type == "json_object":
+            mode = "json_object"
+        elif format_type == "json_schema":
+            mode = "json_schema"
+            json_schema = response_format.get("json_schema")
+            if isinstance(json_schema, dict):
+                schema = json_schema.get("schema") if isinstance(json_schema.get("schema"), dict) else json_schema
+
+    return mode, schema
+
+
+def with_json_contract(messages, mode, schema):
+    if mode == "text":
+        return messages
+
+    if schema:
+        contract = (
+            "Return only a valid JSON object matching this JSON Schema. "
+            "Do not include markdown, code fences, comments, or prose outside the JSON object.\n"
+            + json.dumps(schema, ensure_ascii=False, separators=(",", ":"))
+        )
+    else:
+        contract = (
+            "Return only a valid JSON object. Do not include markdown, code fences, "
+            "comments, or prose outside the JSON object."
+        )
+
+    contracted = [dict(message) for message in messages]
+    for message in contracted:
+        if message["role"] == "system":
+            message["content"] = (message["content"].rstrip() + "\n\n" + contract).strip()
+            return contracted
+
+    contracted.insert(0, {"role": "system", "content": contract})
+    return contracted
+
+
 def render_prompt(messages):
     chat_template = getattr(tokenizer, "chat_template", None)
     if hasattr(tokenizer, "apply_chat_template") and chat_template:
@@ -1219,6 +1785,142 @@ def generate_completion(messages, max_tokens, temperature):
     return content, prompt_tokens, int(completion_ids.shape[-1])
 
 
+def extract_json_value(content):
+    decoder = json.JSONDecoder()
+    text = content.strip()
+    try:
+        return json.loads(text), None
+    except Exception:
+        pass
+
+    for index, char in enumerate(text):
+        if char not in "{[":
+            continue
+        try:
+            value, end = decoder.raw_decode(text[index:])
+            trailing = text[index + end :].strip()
+            if trailing:
+                continue
+            return value, None
+        except Exception:
+            continue
+    return None, "response is not valid JSON"
+
+
+def json_type_matches(value, expected):
+    if expected == "object":
+        return isinstance(value, dict)
+    if expected == "array":
+        return isinstance(value, list)
+    if expected == "string":
+        return isinstance(value, str)
+    if expected == "boolean":
+        return isinstance(value, bool)
+    if expected == "integer":
+        return isinstance(value, int) and not isinstance(value, bool)
+    if expected == "number":
+        return (isinstance(value, int) or isinstance(value, float)) and not isinstance(value, bool)
+    if expected == "null":
+        return value is None
+    return True
+
+
+def validate_json_schema(value, schema, path="$"):
+    if not isinstance(schema, dict):
+        return []
+
+    errors = []
+    expected_type = schema.get("type")
+    if isinstance(expected_type, list):
+        if not any(json_type_matches(value, item) for item in expected_type):
+            errors.append(f"{path} has wrong type")
+            return errors
+    elif isinstance(expected_type, str) and not json_type_matches(value, expected_type):
+        errors.append(f"{path} must be {expected_type}")
+        return errors
+
+    if "enum" in schema and value not in schema["enum"]:
+        errors.append(f"{path} must be one of {schema['enum']}")
+
+    if isinstance(value, dict):
+        required = schema.get("required") if isinstance(schema.get("required"), list) else []
+        for key in required:
+            if key not in value:
+                errors.append(f"{path}.{key} is required")
+
+        properties = schema.get("properties") if isinstance(schema.get("properties"), dict) else {}
+        for key, child_schema in properties.items():
+            if key in value:
+                errors.extend(validate_json_schema(value[key], child_schema, f"{path}.{key}"))
+
+        if schema.get("additionalProperties") is False:
+            extras = sorted(set(value.keys()) - set(properties.keys()))
+            for key in extras:
+                errors.append(f"{path}.{key} is not allowed")
+
+    if isinstance(value, list) and isinstance(schema.get("items"), dict):
+        for index, item in enumerate(value):
+            errors.extend(validate_json_schema(item, schema["items"], f"{path}[{index}]"))
+
+    return errors
+
+
+def normalize_json_content(content, mode, schema):
+    if mode == "text":
+        return content, []
+
+    value, error = extract_json_value(content)
+    if error:
+        return content, [error]
+    if mode in {"json_object", "json_schema"} and not isinstance(value, dict):
+        return content, ["response must be a JSON object"]
+
+    errors = validate_json_schema(value, schema) if schema else []
+    if errors:
+        return content, errors
+
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":")), []
+
+
+def generate_validated_completion(messages, max_tokens, temperature, mode, schema):
+    contracted_messages = with_json_contract(messages, mode, schema)
+    attempts = max(0, JSON_REPAIR_ATTEMPTS) + 1
+    last_content = ""
+    last_prompt_tokens = 0
+    last_completion_tokens = 0
+    last_errors = []
+
+    for attempt in range(attempts):
+        content, prompt_tokens, completion_tokens = generate_completion(
+            contracted_messages,
+            max_tokens,
+            temperature,
+        )
+        normalized, errors = normalize_json_content(content, mode, schema)
+        if not errors:
+            return normalized, prompt_tokens, completion_tokens, []
+
+        last_content = content
+        last_prompt_tokens = prompt_tokens
+        last_completion_tokens = completion_tokens
+        last_errors = errors
+
+        if attempt < attempts - 1:
+            contracted_messages = contracted_messages + [
+                {"role": "assistant", "content": content},
+                {
+                    "role": "user",
+                    "content": (
+                        "The previous response was invalid: "
+                        + "; ".join(errors[:5])
+                        + ". Return only a corrected JSON object."
+                    ),
+                },
+            ]
+
+    return last_content, last_prompt_tokens, last_completion_tokens, last_errors
+
+
 class Handler(BaseHTTPRequestHandler):
     server_version = "TunedTensorReferenceServer/0.1"
 
@@ -1267,7 +1969,23 @@ class Handler(BaseHTTPRequestHandler):
             max_tokens = int(body.get("max_tokens") or DEFAULT_MAX_TOKENS)
             temperature = float(body.get("temperature") if body.get("temperature") is not None else DEFAULT_TEMPERATURE)
             messages = normalize_messages(raw_messages)
-            content, prompt_tokens, completion_tokens = generate_completion(messages, max_tokens, temperature)
+            json_mode, json_schema = response_json_contract(body)
+            content, prompt_tokens, completion_tokens, validation_errors = generate_validated_completion(
+                messages,
+                max_tokens,
+                temperature,
+                json_mode,
+                json_schema,
+            )
+            if validation_errors:
+                self.send_json(422, {
+                    "error": {
+                        "message": "Model did not produce valid JSON for the requested response_format.",
+                        "details": validation_errors,
+                        "content": content,
+                    }
+                })
+                return
 
             self.send_json(200, {
                 "id": "chatcmpl-" + uuid.uuid4().hex,
@@ -1385,7 +2103,7 @@ async function downloadUrlToFile(url, outputPath) {
   if (!res.body) {
     throw new Error("Download failed: response body was empty");
   }
-  mkdirSync2(dirname(outputPath), { recursive: true });
+  mkdirSync3(dirname2(outputPath), { recursive: true });
   const file = createWriteStream(outputPath);
   const contentLength = parseContentLength(res.headers.get("content-length"));
   const progress = createDownloadProgress(
@@ -1513,7 +2231,7 @@ function directoryHasFiles(path) {
 function extractArchive(archivePath, targetDir, force) {
   if (directoryHasFiles(targetDir) && !force) return targetDir;
   if (existsSync4(targetDir)) rmSync(targetDir, { recursive: true, force: true });
-  mkdirSync2(targetDir, { recursive: true });
+  mkdirSync3(targetDir, { recursive: true });
   execFileSync("tar", ["-xzf", archivePath, "-C", targetDir], { stdio: "inherit" });
   return targetDir;
 }
@@ -1547,9 +2265,21 @@ function loadSystemPromptFromSpec(specPath) {
   const spec = JSON.parse(readFileSync6(specPath, "utf8"));
   return buildSystemPrompt(spec);
 }
+function loadJsonSchemaForServe(schemaPath) {
+  const resolved = resolve2(schemaPath);
+  if (!existsSync4(resolved) || !statSync3(resolved).isFile()) {
+    throw new Error(`JSON schema file not found: ${schemaPath}`);
+  }
+  try {
+    const schema = JSON.parse(readFileSync6(resolved, "utf8"));
+    return JSON.stringify(schema);
+  } catch (err) {
+    throw new Error(`JSON schema file is not valid JSON: ${schemaPath}`);
+  }
+}
 function writeReferenceServerScript(cacheDir) {
   const scriptDir = join2(cacheDir, "_server");
-  mkdirSync2(scriptDir, { recursive: true });
+  mkdirSync3(scriptDir, { recursive: true });
   const scriptPath = join2(scriptDir, "openai_reference_server.py");
   writeFileSync3(scriptPath, REFERENCE_SERVER_SCRIPT, "utf8");
   return scriptPath;
@@ -1596,16 +2326,34 @@ function quoteCommandPart(part) {
   if (/^[a-zA-Z0-9_./:=@-]+$/.test(part)) return part;
   return `'${part.replace(/'/g, "'\\''")}'`;
 }
-function printServeCommand(command, args, env) {
+function parseServeInteger(value, label) {
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < 0) {
+    throw new Error(`${label} must be a non-negative integer.`);
+  }
+  return parsed;
+}
+function printServeCommand(command, args, env, managed) {
   const envKeys = Object.keys(env).sort();
   const commandLine = [command, ...args].map(quoteCommandPart).join(" ");
   if (isJsonMode()) {
-    return printJson({ command, args, env_keys: envKeys, command_line: commandLine });
+    return printJson({ command, args, env_keys: envKeys, command_line: commandLine, managed });
   }
-  printDetail([
+  const fields = [
     ["Command", commandLine],
     ["Environment", envKeys.join(", ")]
-  ]);
+  ];
+  if (managed?.enabled) {
+    fields.push(
+      ["Managed", "enabled"],
+      ["Wrapper", `http://${managed.host}:${managed.port}`],
+      ["Idle timeout", `${managed.idle_timeout_seconds}s`],
+      ["Restart after", managed.restart_after_requests === 0 ? "disabled" : `${managed.restart_after_requests} requests`],
+      ["Gate field", managed.gate_field],
+      ["Log file", managed.log_file]
+    );
+  }
+  printDetail(fields);
 }
 function registerModelsCommands(parent) {
   const models = parent.command("models").description("Manage fine-tuned models");
@@ -1709,7 +2457,7 @@ function registerModelsCommands(parent) {
       rmSync(venvDir, { recursive: true, force: true });
     }
     if (!existsSync4(venvPython)) {
-      mkdirSync2(cacheDir, { recursive: true });
+      mkdirSync3(cacheDir, { recursive: true });
       execFileSync(commands[0][0], commands[0].slice(1), { stdio: "inherit" });
     }
     execFileSync(commands[1][0], commands[1].slice(1), { stdio: "inherit" });
@@ -1726,10 +2474,10 @@ function registerModelsCommands(parent) {
     printSuccess(`Serving runtime ready at ${venvDir}`);
     console.log(`Use it with: tt models serve <model-id> --spec ${DEFAULT_SPEC_FILE}`);
   });
-  models.command("serve").description("Serve a downloaded model with an OpenAI-compatible local API").argument("<target>", "Model ID/prefix, model directory, or .tar.gz artifact").option("--spec <path>", "Behavior spec JSON to apply as the default system prompt").option("--no-spec-prompt", "Do not auto-apply a behavior spec system prompt").option("--host <host>", "Host to bind", "127.0.0.1").option("--port <port>", "Port to bind", "8000").option("--python <path>", "Python executable").option("--cache-dir <path>", "Cache directory for downloaded/extracted models").option("--device <device>", "Inference device: auto, cpu, cuda, or mps", "auto").option("--force-download", "Re-download and re-extract model artifacts").option("--max-tokens <n>", "Default max completion tokens", "512").option("--temperature <n>", "Default sampling temperature", "0.7").option("--trust-remote-code", "Pass trust_remote_code=True to transformers").option("--print-command", "Print the underlying Python command without starting it").action(async (target, cmdOpts) => {
+  models.command("serve").description("Serve a downloaded model with an OpenAI-compatible local API").argument("<target>", "Model ID/prefix, model directory, or .tar.gz artifact").option("--spec <path>", "Behavior spec JSON to apply as the default system prompt").option("--no-spec-prompt", "Do not auto-apply a behavior spec system prompt").option("--host <host>", "Host to bind", "127.0.0.1").option("--port <port>", "Port to bind", "8000").option("--python <path>", "Python executable").option("--cache-dir <path>", "Cache directory for downloaded/extracted models").option("--device <device>", "Inference device: auto, cpu, cuda, or mps", "auto").option("--force-download", "Re-download and re-extract model artifacts").option("--max-tokens <n>", "Default max completion tokens", "512").option("--temperature <n>", "Default sampling temperature", "0.7").option("--json-schema <path>", "JSON Schema file to enforce by default for chat completions").option("--json-repair-attempts <n>", "Number of JSON/schema repair retries", "1").option("--trust-remote-code", "Pass trust_remote_code=True to transformers").option("--managed", "Run a local lifecycle manager in front of the model server").option("--idle-timeout <seconds>", "Managed mode idle timeout before stopping the model", "300").option("--restart-after-requests <n>", "Managed mode restart threshold; 0 disables request-count restarts", "100").option("--gate-field <field>", "Response JSON field to log as the managed serving gate result", "should_process").option("--log-file <path>", "Write managed serving JSONL request logs to a file").option("--print-command", "Print the underlying Python command without starting it").action(async (target, cmdOpts) => {
     const opts = parent.opts();
     const cacheDir = resolve2(cmdOpts.cacheDir || defaultCacheDir());
-    mkdirSync2(cacheDir, { recursive: true });
+    mkdirSync3(cacheDir, { recursive: true });
     const python = resolveServePython(cacheDir, cmdOpts.python);
     if (!cmdOpts.printCommand) ensureServingRuntime(python, cacheDir);
     const serveTarget = await resolveServeTarget(
@@ -1757,14 +2505,33 @@ function registerModelsCommands(parent) {
       TT_PORT: String(cmdOpts.port),
       TT_MAX_TOKENS: String(cmdOpts.maxTokens),
       TT_TEMPERATURE: String(cmdOpts.temperature),
+      TT_JSON_REPAIR_ATTEMPTS: String(cmdOpts.jsonRepairAttempts),
       TT_TRUST_REMOTE_CODE: cmdOpts.trustRemoteCode ? "true" : "false",
       TT_DEVICE: String(cmdOpts.device)
     };
     if (systemPrompt) env.TT_SYSTEM_PROMPT = systemPrompt;
+    if (cmdOpts.jsonSchema) env.TT_JSON_SCHEMA = loadJsonSchemaForServe(cmdOpts.jsonSchema);
     const args = [serverScript];
-    if (cmdOpts.printCommand) return printServeCommand(python, args, env);
+    const idleTimeoutSeconds = parseServeInteger(cmdOpts.idleTimeout, "--idle-timeout");
+    const restartAfterRequests = parseServeInteger(
+      cmdOpts.restartAfterRequests,
+      "--restart-after-requests"
+    );
+    const publicPort = parseServeInteger(cmdOpts.port, "--port");
+    const managedConfig = cmdOpts.managed ? {
+      enabled: true,
+      host: String(cmdOpts.host),
+      port: publicPort,
+      idle_timeout_seconds: idleTimeoutSeconds,
+      restart_after_requests: restartAfterRequests,
+      gate_field: String(cmdOpts.gateField),
+      log_file: cmdOpts.logFile ? resolve2(cmdOpts.logFile) : void 0
+    } : void 0;
+    if (cmdOpts.printCommand) return printServeCommand(python, args, env, managedConfig);
     if (!isJsonMode()) {
-      printSuccess(`Serving ${serveTarget.modelName} from ${serveTarget.modelPath}`);
+      printSuccess(
+        cmdOpts.managed ? `Serving ${serveTarget.modelName} through a managed local wrapper` : `Serving ${serveTarget.modelName} from ${serveTarget.modelPath}`
+      );
       if (systemPrompt && specPath) {
         printSuccess(`Behavior spec prompt loaded from ${specPath}`);
       } else if (cmdOpts.specPrompt !== false) {
@@ -1775,9 +2542,32 @@ function registerModelsCommands(parent) {
       console.log(`OpenAI-compatible endpoint: http://${cmdOpts.host}:${cmdOpts.port}/v1/chat/completions`);
       console.log(`Health check: http://${cmdOpts.host}:${cmdOpts.port}/health`);
       console.log(`Python runtime: ${python}`);
+      if (cmdOpts.managed) {
+        console.log(`Managed idle timeout: ${idleTimeoutSeconds}s`);
+        console.log(
+          `Managed restart threshold: ${restartAfterRequests === 0 ? "disabled" : `${restartAfterRequests} requests`}`
+        );
+        console.log(`Managed gate field: ${cmdOpts.gateField}`);
+      }
+    }
+    if (cmdOpts.managed) {
+      await startManagedModelServer({
+        publicHost: String(cmdOpts.host),
+        publicPort,
+        python,
+        args,
+        env,
+        modelName: serveTarget.modelName,
+        modelPath: serveTarget.modelPath,
+        idleTimeoutSeconds,
+        restartAfterRequests,
+        gateField: String(cmdOpts.gateField),
+        logFile: cmdOpts.logFile ? resolve2(cmdOpts.logFile) : void 0
+      });
+      return;
     }
     await new Promise((resolvePromise, reject) => {
-      const child = spawn(python, args, {
+      const child = spawn2(python, args, {
         stdio: "inherit",
         env: { ...process.env, ...env }
       });
@@ -2091,7 +2881,7 @@ function registerPushCommand(parent) {
 
 // src/index.ts
 var program = new Command();
-program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.13").option("-k, --api-key <key>", "API key (overrides stored key)").option(
+program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.15").option("-k, --api-key <key>", "API key (overrides stored key)").option(
   "-u, --base-url <url>",
   "API base URL (default: https://tunedtensor.com)"
 ).option("--json", "Output raw JSON").option("--no-color", "Disable colors").hook("preAction", (_thisCommand, actionCommand) => {