@tuned-tensor/cli 0.25.0 → 0.26.0

package/dist/{prefetch-GP3OPUUL.js → prefetch-DH365HNN.js}

@@ -6,7 +6,7 @@ import {
   safeName,
   verifyLocalBaseModel,
   verifyModelPrefetchCacheReport
-} from "./chunk-UCJCWJ6F.js";
+} from "./chunk-4BHDPRYD.js";
 export {
   buildModelPrefetchPayload,
   buildModelSourcePrefetchPayload,
@@ -15,4 +15,4 @@ export {
   verifyLocalBaseModel,
   verifyModelPrefetchCacheReport
 };
-//# sourceMappingURL=prefetch-GP3OPUUL.js.map
+//# sourceMappingURL=prefetch-DH365HNN.js.map

package/docs/local-runtime/architecture.md

@@ -121,6 +121,57 @@ The Python evaluator receives only opaque IDs and prompts. Node joins its
 predictions to trusted references before scoring. This avoids accidental label
 leakage across the process boundary.
 
+Generation uses left-padded batches of four prompts by default. Set
+`evaluation.inference.batchSize` (1–128) in `tunedtensor.json` to tune memory
+use. A CUDA allocation failure halves the batch and retries the unfinished
+examples; failure at one example is reported instead of retrying forever.
+Batch latency is the time each request waits for its whole batch, including
+decoding; it is not latency divided by batch size. Batched greedy generation
+can differ from single-prompt generation because of floating-point arithmetic.
+
+On integrated CUDA devices such as GB10, training and evaluation first open
+the weights on CPU, then transfer them to CUDA. This avoids retaining host
+checkpoint copies alongside the full GPU allocation in shared system memory.
+Generation and training still execute on CUDA with the same configured dtype.
+
+Ordinary run output shows loading/generation, completed examples, throughput,
+ETA and a heartbeat every 30 seconds. It warns if the estimated remaining work
+exceeds `evaluation.timeoutMs`; that limit remains a hard wall-clock deadline
+(30 minutes by default), including model loading. An ETA is an estimate, not
+a deadline extension. `--verbose` additionally streams the raw process logs.
+
+Each completed batch atomically saves a private prediction checkpoint. An
+incomplete checkpoint is never scored as a completed evaluation. With the
+default `evaluation.baselineCache: true`, rerunning the same baseline inputs
+in a new run automatically continues the checkpoint in the local store's
+`cache/baseline-evals/` directory. Changing only the timeout preserves reuse.
+Changed prompts, model revision/fingerprint, generation settings, batch size
+or package version invalidate it. Local base-model paths without a verified
+fingerprint do not resume. Adapter contents are hashed before candidate
+checkpoint reuse, so retraining at the same path cannot reuse old answers.
+Per-run checkpoints live in `inference-checkpoints/` in the run directory;
+retrying the same saved run request reuses them. A new training run has a new
+candidate identity. Completed baseline reports retain their existing cache.
+Checkpoints contain generated text; remove them when no longer needed.
+
+For a smoke run, explicitly set a small deterministic sample, for example:
+
+```json
+"evaluation": {
+  "maxExamples": 16,
+  "sampleSeed": 42,
+  "timeoutMs": 3600000,
+  "inference": { "batchSize": 4 }
+}
+```
+
+This limits evaluation, not training. Baseline and candidate use the same
+sample. Remove `maxExamples` for full held-out evaluation and choose a timeout
+appropriate to the measured throughput. TT never silently reduces the
+evaluation set to meet a deadline. A smoke sample is pipeline validation,
+not evidence of overall model quality. Exact text matching is also not a
+coding correctness test; the scoring contract is unchanged here.
+
 ## Storage and integrity
 
 Canonical JSON and JSONL files are the state store; there is no database or

package/docs/local-runtime/aws-gpu.md

@@ -169,6 +169,13 @@ evaluation (`evaluation.inference.device: "cpu"`) stays on the laptop.
 Remove `gpu` to execute everything locally. `--dry-run` or `dryRun: true`
 does not connect to AWS or copy files.
 
+To keep using the terminal or the TT agent during a long run, start it in the
+background instead: `tt pipeline start --spec tunedtensor.json --reason "..."`,
+or approve a background run the agent proposes after the dry-run. Check it with
+`tt pipeline status <id>` and stop it with `tt pipeline cancel <id> --reason "..."`.
+The laptop still orchestrates the run, so keep it awake. See
+[background runs](background-runs.md).
+
 ## 5. Inspect results and finish
 
 Wait for a successful completion message. Adapter runs return a run ID and a
@@ -228,6 +235,14 @@ Foundation pretraining can resume from returned checkpoints using its existing
 so checkpoint identities still match. Adapter training keeps its existing
 restart behavior.
 
+Adapter evaluation runs in configurable batches (`evaluation.inference.batchSize`,
+default 4), reducing the batch after a CUDA out-of-memory error. Completed
+prediction checkpoints return with outputs on success, cancellation or timeout
+while SSH remains reachable. Matching baseline inputs reuse the returned
+checkpoint on the next run when `evaluation.baselineCache` is enabled. As with
+foundation checkpoints, nothing is continuously copied to the laptop during
+generation. See [evaluation and smoke runs](architecture.md#evaluation).
+
 You provide capacity and pay AWS directly. TT does not create, start, stop,
 terminate or resize instances, request quotas, choose fallback instances or
 fall back to another account. An AWS profile alone cannot create capacity.

package/docs/local-runtime/background-runs.md

@@ -0,0 +1,77 @@
+# Background runs
+
+A background run is a real `tt pipeline run` that keeps going after the shell or
+conversation that started it moves on. Start one from the TT agent (after an
+approved dry-run) or directly:
+
+```bash
+tt pipeline run --spec tunedtensor.json --dry-run
+tt pipeline start --spec tunedtensor.json --reason "Check the new negation examples"
+tt pipeline list
+tt pipeline status <id>
+tt pipeline logs <id>
+tt pipeline cancel <id> --reason "Started on the wrong dataset"
+```
+
+IDs accept a unique prefix. `--json` prints machine-readable output for every
+command.
+
+## From the agent
+
+The agent can prepare, but never start, a run. The flow is:
+
+1. `prepare_pipeline_run`: you approve a dry-run preview of the saved spec.
+2. `prepare_background_run`: the agent proposes a real run of **the same spec
+   bytes** with a short reason. The proposal shows the plan, the target (laptop
+   or AWS instance), active runs, the parallel-run limit and current CPU, memory
+   and GPU usage. It is refused if no matching dry-run completed in the
+   conversation, if the spec changed, or if the parallel limit is reached.
+3. On approval, tt revalidates the spec, writes the run record and starts a
+   detached supervisor. The conversation continues immediately.
+
+While a run is active, the agent answers questions with read-only tools:
+`list_background_runs`, `get_background_run` (status, metrics and the recent
+log) and `get_system_usage` (CPU, memory, per-GPU load and active runs). It can
+only stop a run through `prepare_cancel_background_run`, which you approve. When
+a run started from a conversation finishes, the agent is told its outcome on your
+next message; it cannot poll in the background.
+
+## Isolation
+
+The run executes the sealed spec captured at approval, not the live
+`tunedtensor.json`, so later spec edits in the conversation cannot change it.
+The snapshot sits beside the spec while the run is active (as
+`.tunedtensor.json.tt-agent-<id>.json`), so relative paths, output directories
+and `tunedtensor.lock.json` behave exactly as for a direct run. It is removed
+when the run ends. The supervisor runs in its own process group, so closing tt
+or the terminal does not stop the run; use `tt pipeline cancel`.
+
+## Parallel runs
+
+By default one background run may be active at a time. Set
+`TT_MAX_PARALLEL_RUNS` (1–8) to allow more. Runs that share a target (the
+laptop GPU, or the same AWS instance) share its GPU memory: check
+`get_system_usage`, `nvidia-smi` or `tt doctor` first. The proposal warns when a
+new run would share a target with an active one. Laptop usage figures do not
+describe a remote AWS GPU.
+
+## Audit record
+
+Each run keeps a private directory under `~/.tuned-tensor/background-runs/<id>/`:
+
+| File | Contents |
+| --- | --- |
+| `record.json` | Reason; requester (OS user, host, `agent` or `cli`, conversation, action and dry-run action IDs); approval time; choices (engine, spec path and SHA-256, config, steps, target, base model); resources at launch; process IDs; cancellation reason and requester; outcome (exit code, local run ID, model ID, report path, evaluation metrics, error). |
+| `spec.json` | The sealed spec the run consumed. |
+| `events.jsonl` | Hash-chained lifecycle events: requested, started, cancel requested, finished or lost. `tt pipeline status` verifies the chain. |
+| `output.log` | Progress and process logs. |
+| `result.json` | The pipeline's final JSON result. |
+
+Adapter runs also keep their usual record in the local run store; use
+`tt runs audit <run-id>`, `tt runs report <run-id>` and
+`tt runs compare <a> <b>` with the run ID from the outcome. The hash chain makes
+casual edits evident; it is not a signature.
+
+If the supervisor disappears (crash, reboot), the run is reported as `lost` the
+next time it is read. Inspect its log and the local run store before starting
+again; a foundation run can be resumed with `tt pipeline run --resume`.

package/docs/local-runtime/decision.md

@@ -12,6 +12,11 @@ TT fine-tunes [Laya](https://huggingface.co/convaiinnovations/laya)
 encoder with a typed decision head. Every stage uses Laya revision
 `55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851`, the reviewed commit that Laya 0.3.22
 pins. Training and evaluation run on CPU, Apple MPS or CUDA; no GPU is required.
+The startup panel and `tt hardware` show decision capabilities even without a
+spec. A full hardware check probes the decision runtime separately from the
+adapter runtime. Native Windows training is not yet end-to-end certified;
+Intel macOS and Windows ARM64 have no compatible packaged PyTorch wheel.
+See the [platform guidance](../../README.md) for setup and memory caveats.
 
 ## Spec
 

package/docs/local-runtime/serving.md

@@ -28,8 +28,12 @@ not mutate the training runtime.
   Keep each model paired with its original training tokenizer. Context defaults to
   the checkpoint limit. Text completions and chat are supported; foundation tool-call
   parsing and image inputs are not supported.
-- Qwen3.5-2B base and a one-step, rank-16 `all-linear` TT-shaped adapter have been
-  exercised on GB10. The registry's Nemotron and Muse Glimmer architectures have
+- Qwen3.5-2B base and a rank-16 `all-linear` adapter trained for one epoch on
+  10,458 banking-intent examples have been exercised on GB10. On a fixed
+  100-example test sample spanning all 77 labels, serving scored 53/100 for the
+  base and 93/100 for the adapter; all adapter predictions matched offline
+  evaluation. This checks adapter application, not general coding ability.
+  The registry's Nemotron and Muse Glimmer architectures have
   upstream support, but their exact adapters/hardware combinations are **not
   certified by that Qwen smoke test**. No 70B model was tested here.
 
@@ -48,6 +52,11 @@ never enables runtime adapter upload/loading. There is no merge/export step
 and no modification of the verified weights. `tt serve base` starts base-only.
 `active` still fails when no adapter is activated.
 
+Qwen text-model adapters use a temporary copy with layer names mapped to vLLM's
+conditional-model layout. Tensor values, adapter settings and the verified
+artifact stay unchanged. This prevents vLLM from accepting an adapter whose
+layer names do not match the loaded model and silently serving base behavior.
+
 Artifact integrity, certified base revisions, and stored behavior-prompt
 fingerprints retain their verification. A stored adapter's behavior prompt is
 applied to both model IDs for comparison. `tt serve base` without explicit

package/docs/spec-workflow.md

@@ -24,11 +24,17 @@ Inside the TT shell, in your project directory:
 
 `/spec` shows a readable overview: name, engine, base model and example count,
 then highlighted sections for behavior (system prompt, guidelines, constraints),
-examples, training settings, evaluation, runtime and pipeline, followed by
+the `dataset_prebuilt` data source when set, examples, training settings, evaluation, runtime and pipeline, followed by
 validation results. `/spec <section>` shows one section in full; sections are
-`identity`, `behavior`, `examples`, `training`, `evaluation`, `runtime` and
+`identity`, `behavior`, `dataset`, `examples`, `training`, `evaluation`, `runtime` and
 `pipeline`. `/spec show` prints the complete file with JSON highlighting,
 validation results and SHA-256.
+
+For adapter specs, `dataset_prebuilt` replaces inline examples as training data;
+the examples section labels this explicitly. Foundation and decision specs reject
+`dataset_prebuilt`; remove it to validate the spec. Foundation corpus settings
+are shown under `/spec training`.
+
 `/spec diff` shows pending spec edits in the current conversation; otherwise it
 shows the latest approved edit, or external changes since that revision.
 `/spec history` lists up to 50 recent revision records. These commands require
@@ -163,3 +169,67 @@ reproducibility; these are outputs, not additional user-authored configuration.
 Agent `/approve` for a pipeline still runs a preview only. Real training uses
 the direct command above. The spec is the behavior recipe; approval of a spec
 edit is not approval to launch training.
+
+## Splitting an unsplit dataset
+
+Keep the reusable master dataset as one chat JSONL file. For a downloaded
+Hugging Face chat dataset, `inspect_local_dataset` returns a pinned `split_source`
+(`hf://datasets/org/repo@<commit>/data.jsonl`). The inspection and split tools
+accept that reference directly, verifying the recorded checksum without copying
+or downloading the master again. Outputs still go into a new workspace directory.
+The CLI accepts the same reference in place of `data.jsonl` below.
+
+Ask the agent:
+
+> Split data.jsonl into 80% training and 20% validation, using seed 123.
+> Balance metadata.category, save to split-123, and use the files in my spec.
+
+The agent validates the file, previews counts, and asks for approval to create
+new split files. It then reads the spec and proposes a separate
+`dataset_prebuilt` edit using the recorded output paths. Training remains a
+separate step: the agent can preview a dry-run; actual training uses
+`tt pipeline run --spec tunedtensor.json`.
+
+The same operation is available without an agent or account:
+
+```sh
+tt datasets inspect data.jsonl
+tt datasets split data.jsonl --output split-123 \
+  --train 0.8 --validation 0.2 --test 0 --seed 123 \
+  --stratify-by metadata.category --dry-run
+# Remove --dry-run to create the reviewed files.
+```
+
+Defaults are 80/10/10 with seed `42`. `--method ordered` assigns groups by source
+order instead of seeded random membership. Records retain their source order
+within each output; the seed controls membership, not training batch order.
+`--group-by group_id` keeps related examples together; without it, normalized
+user prompts define groups. Repeated prompts cannot cross splits, including
+when using a custom grouping field. Groups containing repeated prompts stay in
+training: TT requires unique prompts within each evaluation split. The preview
+and manifest report these rows as `training_only_rows`. Ratios are approximate at group boundaries;
+requested nonzero splits must contain examples. `--stratify-by` accepts a scalar
+field (including dotted metadata paths), and each group must belong to one
+stratum. This is not a near-duplicate detector or a time-series split engine.
+
+Inputs must use an optional system message, one user message, and one assistant
+answer per row. Metadata is preserved. During adapter training, rows without a
+system message receive the current spec’s compiled system prompt, guidelines and
+constraints in the run artifact. An explicit system message must match those
+instructions; it is never silently replaced. Master and split files stay unchanged. Paths must be inside the current
+workspace without symlinks; the output directory must be new, and its parent
+must exist. The source and spec remain unchanged by the splitter. A manifest
+records the source checksum, options, counts, and relative `dataset_prebuilt`
+paths. A prepared agent action refuses a source changed since review.
+
+Source split labels are metadata only. Random/ordered splitting does not enforce
+original benchmark holdouts; preserve those holdouts separately when reporting
+results against the original benchmark. Never shuffle a published test set into
+training and still describe the result as that benchmark's evaluation.
+
+Inspection and splitting stream source records, but the existing adapter training
+loader still materializes complete split files and compiled JSONL in memory.
+Splitting a multi-gigabyte master does not remove that training-loader limit.
+Do not treat a successful split preview as proof that an arbitrarily large split
+can train; streaming training ingestion or bounded dataset selection is separate
+work. Use `tt validate` on the saved spec to validate the actual training inputs.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tuned-tensor/cli",
-  "version": "0.25.0",
+  "version": "0.26.0",
   "description": "Local TT agent and CUDA fine-tuning CLI for Tuned Tensor workflows",
   "repository": {
     "type": "git",
@@ -8,7 +8,8 @@
   },
   "type": "module",
   "bin": {
-    "tt": "dist/index.js"
+    "tt": "dist/index.js",
+    "TT": "dist/index.js"
   },
   "scripts": {
     "build": "tsup",

package/training/adapter/src/evaluate.py

@@ -1,12 +1,13 @@
 from __future__ import annotations
 
 import argparse
+import hashlib
 import json
 import os
 import tarfile
 import time
 from pathlib import Path
-from tempfile import TemporaryDirectory
+from tempfile import NamedTemporaryFile, TemporaryDirectory
 from typing import Any
 
 from model_contract import (
@@ -15,6 +16,7 @@ from model_contract import (
     assert_certified_base_model_revision,
     assert_certified_model_config,
     chat_template_kwargs,
+    cuda_model_load_device_map,
     uncertified_load_kwargs,
 )
 
@@ -186,6 +188,8 @@ def load_text_model(
         raise ValueError(f"{base_model} tokenizer has no EOS token")
     if tokenizer.pad_token_id is None:
         tokenizer.pad_token = tokenizer.eos_token
+    # Decoder-only generation must end every padded prompt on a real token.
+    tokenizer.padding_side = "left"
 
     dtype = None
     if device == "cuda":
@@ -195,7 +199,7 @@ def load_text_model(
     if dtype is not None:
         model_kwargs["dtype"] = dtype
     if device == "cuda":
-        model_kwargs["device_map"] = {"": torch.cuda.current_device()}
+        model_kwargs["device_map"] = cuda_model_load_device_map(torch)
 
     model_class = (
         AutoModelForImageTextToText
@@ -206,7 +210,7 @@ def load_text_model(
     assert_certified_model_config(model.config, expected_model_id=base_model)
     if adapter_path:
         model = PeftModel.from_pretrained(model, adapter_path)
-    if device != "cuda":
+    if device != "cuda" or model_kwargs["device_map"] is None:
         model = model.to(device)
     model.eval()
     return model, tokenizer, device
@@ -223,19 +227,19 @@ def sampling_kwargs(generation: dict[str, Any]) -> dict[str, Any]:
     }
 
 
-def generate_text_one(
+def generate_text_batch(
     model: Any,
     tokenizer: Any,
     system: str,
-    example: dict[str, Any],
+    examples: list[dict[str, Any]],
     generation: dict[str, Any],
     base_model: str = CERTIFIED_BASE_MODEL,
-) -> dict[str, Any]:
-    if example.get("input_assets"):
+) -> list[dict[str, Any]]:
+    if any(example.get("input_assets") for example in examples):
         raise ValueError("The bundled SFT evaluator is text-only and does not accept input_assets")
-    prompt = str(example["input"])
-    formatted = format_prompt(tokenizer, system, prompt, base_model)
-    inputs = tokenizer(formatted, return_tensors="pt")
+    formatted = [format_prompt(tokenizer, system, str(example["input"]), base_model)
+                 for example in examples]
+    inputs = tokenizer(formatted, return_tensors="pt", padding=True, add_special_tokens=False)
     target_device = next(model.parameters()).device
     inputs = {key: value.to(target_device) for key, value in inputs.items()}
     started = time.perf_counter()
@@ -243,24 +247,126 @@ def generate_text_one(
         generated = model.generate(
             **inputs,
             max_new_tokens=int(generation.get("max_new_tokens", 256)),
-            pad_token_id=tokenizer.eos_token_id,
+            pad_token_id=tokenizer.pad_token_id,
             **sampling_kwargs(generation),
         )
-    latency_ms = max(0, round((time.perf_counter() - started) * 1000))
     input_length = int(inputs["input_ids"].shape[-1])
-    output_tokens = generated[0][input_length:]
-    actual = tokenizer.decode(output_tokens, skip_special_tokens=True).strip()
-    return {
+    # Slice at the padded width, not each prompt's unpadded length.
+    actuals = [tokenizer.decode(row[input_length:], skip_special_tokens=True).strip()
+               for row in generated]
+    # Decoding synchronizes CUDA before measuring wall latency.
+    latency_ms = max(0, round((time.perf_counter() - started) * 1000))
+    return [{
         "id": str(example["id"]),
         "actual": actual,
+        # Each request waits for the whole batch; this is batch wall latency.
         "latency_ms": latency_ms,
-    }
+    } for example, actual in zip(examples, actuals, strict=True)]
+
+
+def write_json_atomic(path: Path, value: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    temporary: str | None = None
+    try:
+        with NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent,
+                                prefix=f".{path.name}.", delete=False) as handle:
+            temporary = handle.name
+            json.dump(value, handle)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, path)
+    finally:
+        if temporary and os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+def checkpoint_identity(payload: dict[str, Any], adapter_path: str | None) -> str:
+    digest = hashlib.sha256(str(payload.get("checkpoint_key", "")).encode())
+    # A retrained/replaced adapter at the same path must never reuse predictions.
+    if adapter_path:
+        root = Path(adapter_path)
+        for path in sorted(root.rglob("*")):
+            if path.is_file():
+                digest.update(str(path.relative_to(root)).encode())
+                with path.open("rb") as handle:
+                    for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                        digest.update(chunk)
+    return digest.hexdigest()
+
+
+def read_checkpoint(path: Path, identity: str, examples: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    try:
+        value = load_json(path)
+        results = value.get("results")
+        if value.get("identity") != identity or not isinstance(results, list) or len(results) > len(examples):
+            return []
+        for example, result in zip(examples, results):
+            if (not isinstance(result, dict) or result.get("id") != str(example["id"])
+                    or not isinstance(result.get("actual"), str)
+                    or type(result.get("latency_ms")) is not int or result["latency_ms"] < 0):
+                return []
+        return results
+    except (OSError, ValueError):
+        # A missing, interrupted or incompatible checkpoint is a fresh evaluation.
+        return []
+
+
+def emit_progress(phase: str, completed: int, total: int, batch_size: int,
+                  elapsed: float = 0, eta: float | None = None) -> None:
+    print("@@tt-evaluation " + json.dumps({
+        "phase": phase, "completed": completed, "total": total,
+        "batch_size": batch_size, "elapsed_seconds": round(elapsed, 2),
+        "eta_seconds": round(eta, 2) if eta is not None else None,
+    }), flush=True)
+
+
+def evaluate_batches(payload: dict[str, Any], adapter_path: str | None,
+                     checkpoint: Path, model_loader: str) -> list[dict[str, Any]]:
+    examples = payload.get("examples", [])
+    batch_size = payload.get("batch_size", 4)
+    if type(batch_size) is not int or not 1 <= batch_size <= 128:
+        raise ValueError("batch_size must be an integer between 1 and 128")
+    generation = payload.get("generation", {})
+    if not isinstance(generation, dict):
+        raise ValueError("generation must be a JSON object")
+    identity = checkpoint_identity(payload, adapter_path)
+    results = read_checkpoint(checkpoint, identity, examples) if payload.get("checkpoint_key") else []
+    resumed = len(results)
+    emit_progress("resuming" if resumed else "loading", resumed, len(examples), batch_size)
+    if resumed == len(examples):
+        return results
+    model, tokenizer, device = load_text_model(payload, adapter_path, model_loader)
+    started = time.perf_counter()
+    emit_progress("generating", resumed, len(examples), batch_size)
+    while len(results) < len(examples):
+        batch = examples[len(results):len(results) + batch_size]
+        predictions = None
+        try:
+            predictions = generate_text_batch(model, tokenizer, str(payload.get("system", "")),
+                                              batch, generation, str(payload["base_model"]))
+        except torch.cuda.OutOfMemoryError:
+            if device != "cuda" or len(batch) == 1:
+                raise
+            # Retry only allocation failures; release the failed call's tensors first.
+            batch_size = max(1, len(batch) // 2)
+        if predictions is None:
+            # The exception's traceback is now released, including failed tensors.
+            torch.cuda.empty_cache()
+            emit_progress("reducing_batch", len(results), len(examples), batch_size)
+            continue
+        results.extend(predictions)
+        write_json_atomic(checkpoint, {"identity": identity, "results": results})
+        elapsed = time.perf_counter() - started
+        eta = elapsed / (len(results) - resumed) * (len(examples) - len(results))
+        emit_progress("generating", len(results), len(examples), batch_size, elapsed, eta)
+    return results
 
 
 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("--input", required=True)
     parser.add_argument("--output", required=True)
+    parser.add_argument("--checkpoint")
     args = parser.parse_args()
 
     payload = load_json(Path(args.input))
@@ -273,37 +379,24 @@ def main() -> None:
             "expected causal_lm or image_text_to_text"
         )
     configure_hugging_face_cache(payload.get("model_cache"))
+    emit_progress("initializing", 0, len(payload.get("examples", [])), payload.get("batch_size", 4))
     import_runtime_dependencies()
 
     with TemporaryDirectory(prefix="tt-local-evaluate-") as tmp_dir:
         adapter_path = resolve_adapter_path(payload.get("adapter_path"), Path(tmp_dir))
-        generation = payload.get("generation", {})
-        if not isinstance(generation, dict):
-            raise ValueError("generation must be a JSON object")
-        model, tokenizer, _device = load_text_model(payload, adapter_path, model_loader)
-        results = [
-            generate_text_one(
-                model,
-                tokenizer,
-                str(payload.get("system", "")),
-                example,
-                generation,
-                str(payload["base_model"]),
-            )
-            for example in payload.get("examples", [])
-        ]
+        checkpoint = Path(args.checkpoint or f"{args.output}.checkpoint.json")
+        results = evaluate_batches(payload, adapter_path, checkpoint, model_loader)
 
     output = {
         "provider": "transformers",
         "model_id": payload.get("model_id"),
         "base_model": payload.get("base_model"),
         "adapter_path": payload.get("adapter_path"),
-        "generation_config": generation,
+        "generation_config": payload.get("generation", {}),
         "results": results,
     }
     output_path = Path(args.output)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    output_path.write_text(json.dumps(output, indent=2), encoding="utf-8")
+    write_json_atomic(output_path, output)
 
 
 if __name__ == "__main__":

package/training/adapter/src/model_contract.py

@@ -20,6 +20,19 @@ CERTIFIED_BASE_MODELS = (CERTIFIED_BASE_MODEL, NEMOTRON_BASE_MODEL, MUSE_GLIMMER
 # the run pins an immutable commit. Node enforces the same contract.
 HUGGING_FACE_REPO_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,95}")
 COMMIT_SHA = re.compile(r"[0-9a-fA-F]{40}")
+
+
+def cuda_model_load_device_map(torch_runtime: Any) -> dict[str, int] | None:
+    """Stage on CPU before CUDA transfer when host and GPU share memory.
+
+    Direct CUDA loading can retain a second host copy until checkpoint readers
+    close. CPU-first loading lets those readers close before the CUDA transfer.
+    """
+    index = torch_runtime.cuda.current_device()
+    integrated = torch_runtime.cuda.get_device_properties(index).is_integrated
+    return None if integrated else {"": index}
+
+
 CERTIFIED_QWEN_TEXT_CONFIG = {
     "model_type": "qwen3_5_text",
     "hidden_size": 2048,

package/training/adapter/src/serve.py

@@ -63,6 +63,35 @@ def pin_native_content_format(args: list[str], model_source: str) -> None:
     args += ["--chat-template-content-format", content_format]
 
 
+def prepare_serving_adapter(adapter_path: str, temp_dir: Path) -> str:
+    # Transformers trains Qwen's text-only model, but vLLM loads the repository's
+    # conditional architecture, even with --language-model-only. Its LoRA mapper
+    # expects the language_model prefix; otherwise it silently misses our layers.
+    if os.environ.get("TT_BASE_MODEL") != "Qwen/Qwen3.5-2B":
+        return adapter_path
+    from safetensors import safe_open
+    from safetensors.torch import save_file
+
+    source = Path(adapter_path)
+    old = "base_model.model.model.layers."
+    new = "base_model.model.model.language_model.layers."
+    with safe_open(str(source / "adapter_model.safetensors"), framework="pt", device="cpu") as weights:
+        if not any(key.startswith(old) for key in weights.keys()):
+            return adapter_path
+        tensors = {}
+        for key in weights.keys():
+            mapped = new + key[len(old):] if key.startswith(old) else key
+            if mapped in tensors:
+                raise ValueError("Adapter contains conflicting text and conditional layer names.")
+            tensors[mapped] = weights.get_tensor(key)
+        destination = temp_dir / "serving-adapter"
+        destination.mkdir()
+        save_file(tensors, str(destination / "adapter_model.safetensors"), metadata=weights.metadata())
+    # Keep the verified archive and extracted training layout unchanged.
+    (destination / "adapter_config.json").write_bytes((source / "adapter_config.json").read_bytes())
+    return str(destination)
+
+
 def build_vllm_args(model_source: str, adapter_path: str | None, temp_dir: Path) -> list[str]:
     args = [
         "serve", model_source,
@@ -101,6 +130,7 @@ def build_vllm_args(model_source: str, adapter_path: str | None, temp_dir: Path)
         adapter_base = config.get("base_model_name_or_path")
         if adapter_base not in {os.environ["TT_BASE_MODEL"], model_source}:
             raise ValueError("The adapter base does not match the verified serving model.")
+        adapter_path = prepare_serving_adapter(adapter_path, temp_dir)
         base_name = "base:" + os.environ["TT_BASE_MODEL"]
         args[args.index("--served-model-name") + 1] = base_name
         args += ["--enable-lora", "--max-lora-rank", str(max(8, int(config["r"]))),