@tuned-tensor/cli 0.24.0 → 0.25.0

package/dist/prefetch-GP3OPUUL.js

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+import {
+  buildModelPrefetchPayload,
+  buildModelSourcePrefetchPayload,
+  prefetchBaseModel,
+  safeName,
+  verifyLocalBaseModel,
+  verifyModelPrefetchCacheReport
+} from "./chunk-UCJCWJ6F.js";
+export {
+  buildModelPrefetchPayload,
+  buildModelSourcePrefetchPayload,
+  prefetchBaseModel,
+  safeName,
+  verifyLocalBaseModel,
+  verifyModelPrefetchCacheReport
+};
+//# sourceMappingURL=prefetch-GP3OPUUL.js.map

package/dist/prefetch-GP3OPUUL.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/docs/local-runtime/architecture.md

@@ -173,7 +173,7 @@ whose stderr is a pipe.
 
 Dataset runs resolve the cached snapshot in Node without network access.
 Each split file must link into the repo's blob store, and its digest must match
-the blob name. With `columns`, records are converted to chat rows under
+the blob name. With `columns`, JSONL or Parquet records are converted to chat rows under
 `<artifactRoot>/datasets/huggingface/`. The conversion path is keyed by the
 source digests, the mapping and the system message.
 

package/docs/spec-workflow.md

@@ -123,12 +123,26 @@ Foundation supports `runtime.artifactRoot` and `runtime.gpu`; its evaluation
 and training settings remain under `foundation`.
 
 `dataset_prebuilt.huggingface` names a Hugging Face dataset repo, an optional
-pinned `revision` and optional `columns` (`input`/`output` field names). Its
-split paths are then files inside that repo. `tt datasets prefetch` downloads
-them, and `tt pipeline run` downloads them automatically if they are missing.
+pinned `revision` and optional `columns` (`input`/`output` field names, or
+`messages` for a chat column). Its split paths are then `.jsonl` or `.parquet`
+files inside that repo; Parquet requires `columns`. `tt datasets prefetch`
+downloads them (the agent's `prepare_download` runs it after approval), and `tt pipeline run` downloads them automatically if they are missing.
+`prepare_download` and `--repo` also download sources the spec does not name yet;
+adopt them with a spec edit afterwards. Spec downloads and the first real run
+write `tunedtensor.lock.json` beside the spec, which pins the dataset and
+base-model commit and file digests; later runs stop when they disagree with it.
 Runs record the resolved commit and file digests in the report. See the README
 section "Train on a Hugging Face dataset".
 
+In the agent, ask which datasets are downloaded, then ask to use one in the
+spec. `list_local_datasets` finds TT's local download records;
+`inspect_local_dataset` reads actual cached columns and a bounded sample.
+The agent can then prepare a `dataset_prebuilt` edit with the exact revision,
+training and held-out filenames, and column mapping, without asking you to
+paste a manifest or run a schema command. The edit still requires approval.
+Inline examples remain in the spec but are not automatically added to prebuilt
+training data. A training-only download needs a distinct evaluation split.
+
 Only `tunedtensor.json` is required. `tt pipeline init --spec tunedtensor.json`
 embeds an advanced recipe when needed; omit this step to keep automatic derivation.
 An explicit `--file` cannot override a conflicting embedded recipe.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tuned-tensor/cli",
-  "version": "0.24.0",
+  "version": "0.25.0",
   "description": "Local TT agent and CUDA fine-tuning CLI for Tuned Tensor workflows",
   "repository": {
     "type": "git",
@@ -62,6 +62,8 @@
     "chalk": "^6.0.0",
     "cli-table3": "^0.6.5",
     "commander": "^15.0.0",
+    "hyparquet": "1.31.4",
+    "hyparquet-compressors": "1.1.2",
     "open": "^10.1.0",
     "ora": "^9.3.0",
     "typebox": "1.3.27",
@@ -69,6 +71,7 @@
   },
   "devDependencies": {
     "@types/node": "^25.3.5",
+    "hyparquet-writer": "0.16.12",
     "tsup": "^8.5.1",
     "tsx": "^4.23.12",
     "typescript": "^5.9.3",

package/training/adapter/src/prefetch.py

@@ -137,6 +137,16 @@ def verify_snapshot(snapshot: Path, expected_model_id: str | None = None) -> tup
     return files, verified_blobs
 
 
+def check_requested_revision(base_model: str, revision: str | None, resolve_latest: bool) -> None:
+    """Certified models keep their reviewed revision. An uncertified model needs
+    a pinned commit, except for a direct download that resolves its latest
+    commit (`tt models prefetch --repo`); the resolved SHA is reported so the
+    spec can pin it."""
+    if resolve_latest and revision is None and not is_certified_base_model(base_model):
+        return
+    assert_certified_base_model_revision(base_model, revision, "Prefetch base model revision")
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(description="Prefetch a TT Hugging Face base model for local use.")
     parser.add_argument("--input", required=True)
@@ -147,11 +157,7 @@ def main() -> None:
     base_model = str(payload["base_model"])
     assert_certified_base_model(base_model, "Prefetch base model")
     requested_revision = payload.get("revision")
-    assert_certified_base_model_revision(
-        base_model,
-        requested_revision,
-        "Prefetch base model revision",
-    )
+    check_requested_revision(base_model, requested_revision, bool(payload.get("resolve_latest")))
     cache_dir = payload.get("model_cache")
     configure_hugging_face_cache(cache_dir)