@tuned-tensor/cli 0.4.20 → 0.4.21

package/dist/index.js

@@ -387,6 +387,7 @@ var SUPPORTED_BASE_MODELS = [
   "google/gemma-4-E2B-it",
   "google/gemma-4-E4B-it",
   "Qwen/Qwen3.5-2B",
+  "Qwen/Qwen3-VL-2B-Instruct",
   "Qwen/Qwen3.5-4B",
   "meta-llama/Llama-3.2-3B-Instruct",
   "microsoft/Phi-4-mini-instruct",
@@ -407,6 +408,9 @@ for (const [alias, model] of [
   ["qwen/qwen3.5-2b", "Qwen/Qwen3.5-2B"],
   ["Qwen/Qwen3.5-2B-Base", "Qwen/Qwen3.5-2B"],
   ["qwen/qwen3.5-2b-base", "Qwen/Qwen3.5-2B"],
+  ["qwen/qwen3-vl-2b-instruct", "Qwen/Qwen3-VL-2B-Instruct"],
+  ["Qwen/Qwen3-VL-2B", "Qwen/Qwen3-VL-2B-Instruct"],
+  ["qwen/qwen3-vl-2b", "Qwen/Qwen3-VL-2B-Instruct"],
   ["qwen/qwen3.5-4b", "Qwen/Qwen3.5-4B"],
   ["Qwen/Qwen3.5-4B-Base", "Qwen/Qwen3.5-4B"],
   ["qwen/qwen3.5-4b-base", "Qwen/Qwen3.5-4B"],
@@ -959,6 +963,7 @@ Eval Results (${data._evals.length}):`);
 
 // src/commands/datasets.ts
 import { existsSync as existsSync2, readFileSync as readFileSync4, statSync as statSync2 } from "fs";
+var DOCUMENT_OCR_MIME_TYPES = /* @__PURE__ */ new Set(["image/png", "image/jpeg", "image/webp"]);
 function formatBytes(bytes) {
   if (bytes < 1024) return `${bytes} B`;
   if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
@@ -1012,10 +1017,85 @@ function findControlChar(value) {
 function formatCodepoint(code) {
   return `U+${code.toString(16).toUpperCase().padStart(4, "0")}`;
 }
-function validateDatasetFile(file) {
+function detectRowFormat(record) {
+  const input = record.input;
+  if (input && typeof input === "object" && typeof input.prompt === "string" && Array.isArray(input.assets) && input.assets.length > 0 && typeof record.output === "string") {
+    return "document_ocr_jsonl";
+  }
+  if (typeof record.input === "string" || typeof record.output === "string" || !("input" in record) || !("output" in record)) {
+    return "jsonl";
+  }
+  return null;
+}
+function validateDocumentOcrRow(record, rowNumber, errors) {
+  const input = record.input;
+  const assets = input.assets;
+  if (typeof input.prompt !== "string" || input.prompt.trim().length === 0) {
+    errors.push(`Row ${rowNumber}: OCR input.prompt must be a non-empty string`);
+  } else {
+    const hit = findControlChar(input.prompt);
+    if (hit) {
+      errors.push(
+        `Row ${rowNumber}: OCR input.prompt contains invisible control character ${formatCodepoint(hit.codepoint)} (${hit.name}) at offset ${hit.offset} \u2014 strip or escape before uploading; some training paths split on it`
+      );
+    }
+  }
+  if (typeof record.output !== "string" || record.output.trim().length === 0) {
+    errors.push(`Row ${rowNumber}: OCR output must be a non-empty string`);
+  } else {
+    const hit = findControlChar(record.output);
+    if (hit) {
+      errors.push(
+        `Row ${rowNumber}: OCR output contains invisible control character ${formatCodepoint(hit.codepoint)} (${hit.name}) at offset ${hit.offset} \u2014 strip or escape before uploading; some training paths split on it`
+      );
+    }
+  }
+  if (assets.length > 8) {
+    errors.push(`Row ${rowNumber}: OCR rows support at most 8 image assets`);
+  }
+  for (const [assetIndex, asset] of assets.entries()) {
+    if (!asset || typeof asset !== "object" || Array.isArray(asset)) {
+      errors.push(`Row ${rowNumber}: asset ${assetIndex + 1} must be an object`);
+      continue;
+    }
+    const a = asset;
+    if (a.type !== void 0 && a.type !== "image") {
+      errors.push(`Row ${rowNumber}: asset ${assetIndex + 1} type must be "image"`);
+    }
+    if (a.mime_type !== void 0 && (typeof a.mime_type !== "string" || !DOCUMENT_OCR_MIME_TYPES.has(a.mime_type))) {
+      errors.push(
+        `Row ${rowNumber}: asset ${assetIndex + 1} mime_type must be one of image/png, image/jpeg, image/webp`
+      );
+    }
+    if (a.page !== void 0 && (typeof a.page !== "number" || !Number.isInteger(a.page) || a.page < 1)) {
+      errors.push(`Row ${rowNumber}: asset ${assetIndex + 1} page must be a positive integer`);
+    }
+    const hasReference = typeof a.data_uri === "string" && a.data_uri.trim().length > 0 || typeof a.uri === "string" && a.uri.trim().length > 0 || typeof a.path === "string" && a.path.trim().length > 0;
+    if (!hasReference) {
+      errors.push(`Row ${rowNumber}: asset ${assetIndex + 1} must include data_uri, uri, or path`);
+    }
+    if (typeof a.data_uri === "string") {
+      const match = /^data:(image\/(?:png|jpeg|webp));base64,[A-Za-z0-9+/=\s]+$/.exec(a.data_uri);
+      if (!match) {
+        errors.push(
+          `Row ${rowNumber}: asset ${assetIndex + 1} data_uri must be a base64 image/png, image/jpeg, or image/webp data URI`
+        );
+      } else if (typeof a.mime_type === "string" && a.mime_type !== match[1]) {
+        errors.push(`Row ${rowNumber}: asset ${assetIndex + 1} mime_type does not match data_uri media type`);
+      }
+    }
+  }
+}
+function parseDatasetFormat(value) {
+  if (!value) return void 0;
+  if (value === "jsonl" || value === "document_ocr_jsonl") return value;
+  throw new Error("--format must be one of: jsonl, document_ocr_jsonl");
+}
+function validateDatasetFile(file, requestedFormat) {
   const lines = readFileSync4(file, "utf8").split(/\r?\n/);
   const errors = [];
   let rowCount = 0;
+  let detectedFormat = null;
   for (const [index, rawLine] of lines.entries()) {
     const line = rawLine.trim();
     if (!line) continue;
@@ -1039,6 +1119,26 @@ function validateDatasetFile(file) {
       );
       continue;
     }
+    const rowFormat = detectRowFormat(record);
+    if (!rowFormat) {
+      errors.push(
+        `Row ${rowNumber}: expected text {"input": string, "output": string} or OCR {"input":{"prompt": string, "assets": [...]}, "output": string}`
+      );
+      continue;
+    }
+    detectedFormat ??= rowFormat;
+    if (requestedFormat && rowFormat !== requestedFormat) {
+      errors.push(`Row ${rowNumber}: expected ${requestedFormat} row, found ${rowFormat}`);
+      continue;
+    }
+    if (detectedFormat !== rowFormat) {
+      errors.push(`Row ${rowNumber}: cannot mix ${detectedFormat} and ${rowFormat} rows in one dataset`);
+      continue;
+    }
+    if (rowFormat === "document_ocr_jsonl") {
+      validateDocumentOcrRow(record, rowNumber, errors);
+      continue;
+    }
     if (typeof record.input !== "string") {
       errors.push(`Row ${rowNumber}: missing string "input" field`);
     }
@@ -1064,10 +1164,11 @@ function validateDatasetFile(file) {
     const suffix = errors.length > 5 ? `
 ...and ${errors.length - 5} more error(s)` : "";
     throw new Error(
-      `Invalid dataset format. Each JSONL row must be {"input": "...", "output": "..."}.
+      `Invalid dataset format. Each JSONL row must be a valid text or document OCR example.
 ${preview}${suffix}`
     );
   }
+  return detectedFormat ?? requestedFormat ?? "jsonl";
 }
 function registerDatasetsCommands(parent) {
   const datasets = parent.command("datasets").description("Manage datasets");
@@ -1115,13 +1216,13 @@ function registerDatasetsCommands(parent) {
       }
     }
   });
-  datasets.command("upload").description("Upload a JSONL dataset file").argument("<file>", "Path to JSONL file").option("-n, --name <name>", "Dataset name (defaults to filename)").option("-d, --description <desc>", "Dataset description").action(async (file, cmdOpts) => {
+  datasets.command("upload").description("Upload a JSONL dataset file").argument("<file>", "Path to JSONL file").option("-n, --name <name>", "Dataset name (defaults to filename)").option("-d, --description <desc>", "Dataset description").option("--format <format>", "Dataset format: jsonl or document_ocr_jsonl").action(async (file, cmdOpts) => {
     const opts = parent.opts();
     if (!existsSync2(file)) {
       printError(`File not found: ${file}`);
       process.exit(1);
     }
-    validateDatasetFile(file);
+    const format = validateDatasetFile(file, parseDatasetFormat(cmdOpts.format));
     const name = cmdOpts.name || file.split("/").pop().replace(/\.jsonl?$/, "");
     const fileBytes = readFileSync4(file);
     const uploadUrl = await post(
@@ -1131,7 +1232,8 @@ function registerDatasetsCommands(parent) {
         description: cmdOpts.description ?? null,
         filename: file.split("/").pop(),
         size: statSync2(file).size,
-        contentType: "application/jsonl"
+        contentType: "application/jsonl",
+        format
       },
       opts
     );
@@ -2074,7 +2176,10 @@ import json
 import os
 import time
 import uuid
+import base64
+from io import BytesIO
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from urllib.request import urlopen
 
 MODEL_PATH = os.environ["TT_MODEL_PATH"]
 MODEL_NAME = os.environ.get("TT_MODEL_NAME", os.path.basename(MODEL_PATH.rstrip("/")) or "tuned-tensor-model")
@@ -2125,9 +2230,9 @@ def load_model():
         ) from exc
 
     try:
-        processor = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=TRUST_REMOTE_CODE)
-    except Exception:
         processor = AutoProcessor.from_pretrained(MODEL_PATH, trust_remote_code=TRUST_REMOTE_CODE)
+    except Exception:
+        processor = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=TRUST_REMOTE_CODE)
     tokenizer = getattr(processor, "tokenizer", processor)
     if getattr(tokenizer, "pad_token", None) is None:
         tokenizer.pad_token = tokenizer.eos_token
@@ -2144,22 +2249,43 @@ def load_model():
         kwargs["torch_dtype"] = torch.float16
 
     try:
-        model = AutoModelForCausalLM.from_pretrained(MODEL_PATH, **kwargs)
-    except Exception:
         model = AutoModelForImageTextToText.from_pretrained(MODEL_PATH, **kwargs)
+    except Exception:
+        model = AutoModelForCausalLM.from_pretrained(MODEL_PATH, **kwargs)
     if device != "cuda":
         model.to(device)
     model.eval()
-    return torch, tokenizer, model, device
+    return torch, processor, tokenizer, model, device
+
+
+torch, processor, tokenizer, model, DEVICE = load_model()
 
 
-torch, tokenizer, model, DEVICE = load_model()
+def normalize_content_part(part):
+    if not isinstance(part, dict):
+        return {"type": "text", "text": str(part)}
+    part_type = part.get("type")
+    if part_type == "text":
+        return {"type": "text", "text": str(part.get("text", ""))}
+    if part_type in {"image", "image_url"}:
+        image_url = part.get("image_url")
+        url = None
+        if isinstance(image_url, dict):
+            url = image_url.get("url")
+        elif isinstance(image_url, str):
+            url = image_url
+        url = url or part.get("image") or part.get("data_uri") or part.get("uri") or part.get("path")
+        if isinstance(url, str) and url.strip():
+            return {"type": "image", "image": url.strip()}
+    return {"type": "text", "text": json.dumps(part, ensure_ascii=False)}
 
 
-def content_to_text(content):
+def normalize_content(content):
     if isinstance(content, str):
         return content
-    return json.dumps(content, ensure_ascii=False)
+    if isinstance(content, list):
+        return [normalize_content_part(part) for part in content]
+    return str(content) if content is not None else ""
 
 
 def normalize_messages(raw_messages):
@@ -2170,7 +2296,7 @@ def normalize_messages(raw_messages):
         role = item.get("role")
         if role not in {"system", "user", "assistant", "tool"}:
             continue
-        messages.append({"role": role, "content": content_to_text(item.get("content", ""))})
+        messages.append({"role": role, "content": normalize_content(item.get("content", ""))})
 
     if SYSTEM_PROMPT and not any(message["role"] == "system" for message in messages):
         messages.insert(0, {"role": "system", "content": SYSTEM_PROMPT})
@@ -2214,7 +2340,10 @@ def with_json_contract(messages, mode, schema):
     contracted = [dict(message) for message in messages]
     for message in contracted:
         if message["role"] == "system":
-            message["content"] = (message["content"].rstrip() + "\n\n" + contract).strip()
+            content = message.get("content", "")
+            if not isinstance(content, str):
+                content = json.dumps(content, ensure_ascii=False)
+            message["content"] = (content.rstrip() + "\n\n" + contract).strip()
             return contracted
 
     contracted.insert(0, {"role": "system", "content": contract})
@@ -2233,13 +2362,71 @@ def render_prompt(messages):
     return "\n".join(rendered)
 
 
-def generate_completion(messages, max_tokens, temperature):
-    prompt = render_prompt(messages)
-    inputs = tokenizer(prompt, return_tensors="pt")
+def load_image(value):
+    try:
+        from PIL import Image
+    except Exception as exc:
+        raise RuntimeError(
+            "Multimodal requests require Pillow. Install the serving runtime with "
+            "'tt models setup-runtime --force' or install pillow in the selected Python environment."
+        ) from exc
+
+    if value.startswith("data:image/"):
+        _, payload = value.split(",", 1)
+        return Image.open(BytesIO(base64.b64decode(payload))).convert("RGB")
+    if value.startswith("http://") or value.startswith("https://"):
+        with urlopen(value, timeout=30) as response:
+            return Image.open(BytesIO(response.read())).convert("RGB")
+    return Image.open(value).convert("RGB")
+
+
+def resolve_message_images(messages):
+    resolved = []
+    images = []
+    for message in messages:
+        content = message.get("content")
+        if not isinstance(content, list):
+            resolved.append(message)
+            continue
+
+        parts = []
+        for part in content:
+            if not isinstance(part, dict) or part.get("type") != "image":
+                parts.append(part)
+                continue
+            image_ref = part.get("image") or part.get("data_uri") or part.get("uri") or part.get("path")
+            if not isinstance(image_ref, str) or not image_ref.strip():
+                raise RuntimeError("Image content part must include image, image_url.url, data_uri, uri, or path")
+            images.append(load_image(image_ref.strip()))
+            parts.append({"type": "image"})
+        resolved.append({**message, "content": parts})
+    return resolved, images
+
+
+def move_inputs_to_device(inputs, device):
+    moved = {}
+    for key, value in inputs.items():
+        moved[key] = value.to(device) if hasattr(value, "to") else value
+    return moved
+
+
+def prepare_generation_inputs(messages):
+    resolved_messages, images = resolve_message_images(messages)
+    prompt = render_prompt(resolved_messages)
     input_device = getattr(model, "device", None)
     if input_device is None:
         input_device = next(model.parameters()).device
-    inputs = {key: value.to(input_device) for key, value in inputs.items()}
+    if images:
+        if processor is tokenizer:
+            raise RuntimeError("This model artifact did not load a multimodal processor, so image inputs are not supported.")
+        inputs = processor(text=[prompt], images=images, return_tensors="pt")
+    else:
+        inputs = tokenizer(prompt, return_tensors="pt")
+    return move_inputs_to_device(inputs, input_device)
+
+
+def generate_completion(messages, max_tokens, temperature):
+    inputs = prepare_generation_inputs(messages)
 
     generate_kwargs = {
         **inputs,
@@ -2667,7 +2854,7 @@ function ensureServingRuntime(python, cacheDir) {
   const check = `
 import importlib.util
 import json
-required = ["torch", "transformers", "accelerate", "safetensors"]
+required = ["torch", "transformers", "accelerate", "safetensors", "PIL"]
 missing = [name for name in required if importlib.util.find_spec(name) is None]
 print(json.dumps({"missing": missing}))
 `;
@@ -3238,7 +3425,7 @@ function registerModelsCommands(parent) {
     const venvDir = runtimeDir(cacheDir);
     const venvPython = runtimePythonPath(cacheDir);
     const python = pickRuntimePython(cmdOpts.python);
-    const deps = ["torch", "transformers", "accelerate", "safetensors"];
+    const deps = ["torch", "transformers", "accelerate", "safetensors", "pillow"];
     const commands = setupRuntimeCommands(python, venvDir, deps);
     if (cmdOpts.printCommand) {
       const commandLines = commands.map((cmd) => cmd.map(quoteCommandPart).join(" "));
@@ -3426,6 +3613,14 @@ function registerBalanceCommands(parent) {
     const balanceLine = lowBalance ? chalk3.red(formatCents3(balance.balance_cents)) + chalk3.dim(" (low)") : chalk3.bold.green(formatCents3(balance.balance_cents));
     printDetail([
       ["Credits", balanceLine],
+      [
+        "Reserved",
+        balance.reserved_cents > 0 ? formatCents3(balance.reserved_cents) : void 0
+      ],
+      [
+        "Available",
+        balance.reserved_cents > 0 ? formatCents3(balance.available_cents) : void 0
+      ],
       ["Lifetime top-ups", formatCents3(balance.lifetime_topup_cents)]
     ]);
     if (transactions && transactions.length > 0) {
@@ -3768,7 +3963,7 @@ function registerPushCommand(parent) {
 
 // src/index.ts
 var program = new Command();
-program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.20").option("-k, --api-key <key>", "API key (overrides stored key)").option(
+program.name("tt").description("Tuned Tensor CLI \u2014 fine-tune and evaluate LLMs").version("0.4.21").option("-k, --api-key <key>", "API key (overrides stored key)").option(
   "-u, --base-url <url>",
   "API base URL (default: https://tunedtensor.com)"
 ).option("--json", "Output raw JSON").option("--no-color", "Disable colors").hook("preAction", (_thisCommand, actionCommand) => {