@tuned-tensor/local 0.2.9 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/README.md +84 -210
- package/dist/artifacts.d.ts +3 -4
- package/dist/artifacts.js +55 -33
- package/dist/artifacts.js.map +1 -1
- package/dist/compare.d.ts +1 -8
- package/dist/compare.js +1 -23
- package/dist/compare.js.map +1 -1
- package/dist/contracts.d.ts +59 -366
- package/dist/contracts.js +73 -224
- package/dist/contracts.js.map +1 -1
- package/dist/dataset.d.ts +2 -4
- package/dist/dataset.js +31 -125
- package/dist/dataset.js.map +1 -1
- package/dist/doctor.d.ts +2 -3
- package/dist/doctor.js +23 -161
- package/dist/doctor.js.map +1 -1
- package/dist/evaluation.d.ts +11 -71
- package/dist/evaluation.js +212 -572
- package/dist/evaluation.js.map +1 -1
- package/dist/huggingface-cache.d.ts +8 -7
- package/dist/huggingface-cache.js +16 -8
- package/dist/huggingface-cache.js.map +1 -1
- package/dist/index.d.ts +0 -10
- package/dist/index.js +203 -626
- package/dist/index.js.map +1 -1
- package/dist/local-project.d.ts +1 -11
- package/dist/local-project.js +33 -61
- package/dist/local-project.js.map +1 -1
- package/dist/model-registry.d.ts +3 -16
- package/dist/model-registry.js +76 -292
- package/dist/model-registry.js.map +1 -1
- package/dist/model-server.d.ts +1 -2
- package/dist/model-server.js +9 -18
- package/dist/model-server.js.map +1 -1
- package/dist/orchestrator.d.ts +11 -25
- package/dist/orchestrator.js +246 -566
- package/dist/orchestrator.js.map +1 -1
- package/dist/prefetch.d.ts +1 -5
- package/dist/prefetch.js +14 -19
- package/dist/prefetch.js.map +1 -1
- package/dist/process-runner.d.ts +10 -29
- package/dist/process-runner.js +63 -169
- package/dist/process-runner.js.map +1 -1
- package/dist/process-training.d.ts +0 -1
- package/dist/process-training.js +10 -87
- package/dist/process-training.js.map +1 -1
- package/dist/store.d.ts +1 -3
- package/dist/store.js +92 -290
- package/dist/store.js.map +1 -1
- package/docs/architecture.md +87 -152
- package/docs/spark.md +66 -97
- package/examples/dry-runner.json +14 -0
- package/examples/local-runner.json +8 -6
- package/examples/smoke-spec.json +41 -0
- package/package.json +6 -6
- package/training/local-runner/pyproject.toml +0 -5
- package/training/local-runner/src/evaluate.py +89 -234
- package/training/local-runner/src/model_contract.py +47 -0
- package/training/local-runner/src/prefetch.py +18 -4
- package/training/local-runner/src/serve.py +54 -115
- package/training/local-runner/src/sft_data.py +97 -0
- package/training/local-runner/src/train.py +146 -346
- package/training/local-runner/uv.lock +0 -1197
- package/dist/labeling-sanitize.d.ts +0 -31
- package/dist/labeling-sanitize.js +0 -158
- package/dist/labeling-sanitize.js.map +0 -1
- package/dist/labeling.d.ts +0 -155
- package/dist/labeling.js +0 -496
- package/dist/labeling.js.map +0 -1
- package/dist/openrouter.d.ts +0 -29
- package/dist/openrouter.js +0 -66
- package/dist/openrouter.js.map +0 -1
- package/dist/server.d.ts +0 -9
- package/dist/server.js +0 -211
- package/dist/server.js.map +0 -1
- package/docs/local-workflow-remediation-2026-07-13.md +0 -130
- package/docs/local-workflow-ux-review-2026-07-13.md +0 -458
- package/examples/dpo-preferences.jsonl +0 -2
- package/examples/dpo-run-request.json +0 -34
- package/examples/smoke-run-request.json +0 -34
- package/training/local-runner/src/train_dpo.py +0 -286
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,65 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to TT Local will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## Unreleased
|
|
6
|
+
|
|
7
|
+
## 0.3.0 - 2026-07-26
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Added one certified end-to-end local workflow for CUDA LoRA SFT of
|
|
12
|
+
`Qwen/Qwen3.5-2B`, including immutable snapshot prefetch, held-out
|
|
13
|
+
base-versus-tuned evaluation, verified PEFT artifacts, and
|
|
14
|
+
OpenAI-compatible local serving.
|
|
15
|
+
- Added exact Qwen model-contract checks in both TypeScript and Python, plus
|
|
16
|
+
strict text-only chat JSONL validation and assistant-only SFT data tests.
|
|
17
|
+
|
|
18
|
+
### Removed
|
|
19
|
+
|
|
20
|
+
- **Breaking:** reduced TT Local to its end-to-end verified purpose: text LoRA
|
|
21
|
+
SFT of `Qwen/Qwen3.5-2B` on CUDA. Removed DPO, multimodal and continuation
|
|
22
|
+
inputs, arbitrary training/evaluation commands, external model contracts,
|
|
23
|
+
cloud teacher labeling and LLM judging, the classic-ML Study subsystem, and
|
|
24
|
+
the local dashboard.
|
|
25
|
+
- Removed the SQLite metadata mirror. Canonical JSON/JSONL files are now the
|
|
26
|
+
only state store and remain directly inspectable and recoverable.
|
|
27
|
+
- Removed implicit working-directory `.env` loading. Credentials and serving
|
|
28
|
+
keys must be provided explicitly through the process environment.
|
|
29
|
+
- Removed the incomplete detached-run, watch, and remote-cancel surfaces.
|
|
30
|
+
Workflows now run in the foreground so process and state lifecycles cannot
|
|
31
|
+
diverge after a host restart or worker crash.
|
|
32
|
+
|
|
33
|
+
### Changed
|
|
34
|
+
|
|
35
|
+
- Training now computes loss only on assistant answer tokens and truncates the
|
|
36
|
+
prompt before the answer. CUDA training and tokenizer chat-template failures
|
|
37
|
+
fail fast instead of silently changing behavior.
|
|
38
|
+
- Runner, spec, and hyperparameter schemas are strict; obsolete and misspelled
|
|
39
|
+
fields are rejected rather than ignored.
|
|
40
|
+
- Training, evaluation, and serving now always use the bundled locked Python
|
|
41
|
+
project. Removed public runner project, working-directory, provider, and
|
|
42
|
+
environment overrides.
|
|
43
|
+
- Bundled `uv` commands now use `--frozen` and a content-keyed virtual
|
|
44
|
+
environment in the writable user cache instead of modifying the npm package.
|
|
45
|
+
- After prefetch verifies an immutable snapshot, training, evaluation, and
|
|
46
|
+
serving enforce offline Hugging Face cache use and the current Transformers
|
|
47
|
+
`dtype` loading API.
|
|
48
|
+
- Explicit base-model revisions must be immutable 40-character Hugging Face
|
|
49
|
+
commit SHAs; moving branches and tags are rejected.
|
|
50
|
+
- Default holdout sampling is stable across runs of the same behavior spec,
|
|
51
|
+
and baseline cache identity no longer includes the per-run UUID.
|
|
52
|
+
- The supported workflow is now documented and tested as one coherent path:
|
|
53
|
+
initialize, preflight, prepare held-out data, train, compare, verify, and
|
|
54
|
+
serve a local PEFT adapter.
|
|
55
|
+
- Prebuilt datasets with both validation and test files now use validation for
|
|
56
|
+
normal iterative evaluation, preserving test as a separate holdout.
|
|
57
|
+
|
|
58
|
+
### Fixed
|
|
59
|
+
|
|
60
|
+
- Completed process-group teardown when a logged command's direct child exits,
|
|
61
|
+
including an opt-in cleanup mode for successful commands that must not leave
|
|
62
|
+
background descendants running.
|
|
63
|
+
|
|
5
64
|
## 0.2.9 - 2026-07-14
|
|
6
65
|
|
|
7
66
|
### Changed
|
package/README.md
CHANGED
|
@@ -3,272 +3,146 @@
|
|
|
3
3
|
[](https://github.com/tunedtensor/tuned-tensor-local/actions/workflows/ci.yml)
|
|
4
4
|
[](https://www.npmjs.com/package/@tuned-tensor/local)
|
|
5
5
|
|
|
6
|
-
TT Local
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
datasets, model artifacts, events, reports, and dashboard state stay on local
|
|
11
|
-
disk.
|
|
6
|
+
TT Local fine-tunes an open-weight language model on your own NVIDIA GPU. It
|
|
7
|
+
turns a small behavior spec or chat JSONL dataset into a verified LoRA adapter,
|
|
8
|
+
compares the base and tuned model on held-out examples, and can serve the
|
|
9
|
+
adapter through an OpenAI-compatible endpoint.
|
|
12
10
|
|
|
13
|
-
|
|
11
|
+
The current release deliberately has one supported path:
|
|
14
12
|
|
|
15
|
-
|
|
13
|
+
- text supervised fine-tuning (SFT);
|
|
14
|
+
- `Qwen/Qwen3.5-2B`;
|
|
15
|
+
- LoRA/PEFT on CUDA;
|
|
16
|
+
- deterministic exact-match or JSON-field evaluation;
|
|
17
|
+
- local files for datasets, state, reports, and model artifacts.
|
|
16
18
|
|
|
17
|
-
|
|
19
|
+
New model families and training methods should be added only after they have a
|
|
20
|
+
repeatable end-to-end GPU acceptance test.
|
|
18
21
|
|
|
19
|
-
|
|
20
|
-
npm install -g @tuned-tensor/local
|
|
21
|
-
tt-local info
|
|
22
|
-
```
|
|
23
|
-
|
|
24
|
-
The bundled SFT, DPO, and Transformers evaluator path also needs `uv`:
|
|
25
|
-
|
|
26
|
-
```bash
|
|
27
|
-
uv --version
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
Custom training or evaluation workflows can use command entrypoints instead of
|
|
31
|
-
`uv`.
|
|
22
|
+
## Requirements
|
|
32
23
|
|
|
33
|
-
|
|
34
|
-
`
|
|
35
|
-
|
|
24
|
+
- Node.js 22 or later
|
|
25
|
+
- [`uv`](https://docs.astral.sh/uv/)
|
|
26
|
+
- an NVIDIA CUDA host with enough free disk space for the model cache and run
|
|
27
|
+
artifacts
|
|
36
28
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
On an NVIDIA Spark or another CUDA host, initialize both the behavior spec and
|
|
40
|
-
a durable project-local runner config:
|
|
29
|
+
The npm package includes the locked Python training environment.
|
|
41
30
|
|
|
42
31
|
```bash
|
|
43
|
-
|
|
44
|
-
tt-local
|
|
45
|
-
```
|
|
46
|
-
|
|
47
|
-
Edit `tunedtensor.json` to replace the generated system prompt and example.
|
|
48
|
-
TT Local refuses to validate or train the unchanged placeholder so an
|
|
49
|
-
accidental run cannot spend GPU time on it.
|
|
50
|
-
|
|
51
|
-
Then preflight the exact Python environments and paths that the run will use:
|
|
52
|
-
|
|
53
|
-
```bash
|
|
54
|
-
tt-local doctor tunedtensor.json --config local-runner.json
|
|
55
|
-
tt-local validate tunedtensor.json --config local-runner.json
|
|
56
|
-
```
|
|
57
|
-
|
|
58
|
-
`doctor` checks the bundled uv environment, Torch/Transformers/PEFT imports,
|
|
59
|
-
the requested CUDA/MPS device, judge credentials when configured, writable
|
|
60
|
-
artifact/cache/store roots, and available disk space. GPU checks are skipped
|
|
61
|
-
for dry runs and are conditional for CPU/MPS configurations.
|
|
62
|
-
|
|
63
|
-
Download the base model explicitly, then start the run:
|
|
64
|
-
|
|
65
|
-
```bash
|
|
66
|
-
tt-local models prefetch tunedtensor.json --config local-runner.json
|
|
67
|
-
tt-local models verify-base tunedtensor.json --config local-runner.json
|
|
68
|
-
tt-local run tunedtensor.json --config local-runner.json
|
|
32
|
+
npm install -g @tuned-tensor/local
|
|
33
|
+
tt-local info
|
|
69
34
|
```
|
|
70
35
|
|
|
71
|
-
|
|
72
|
-
program is consuming the final JSON. `verify-base` is local-only: it checks
|
|
73
|
-
the cached snapshot structure, indexed weight shards, tokenizer files, and
|
|
74
|
-
non-empty weights without downloading a replacement. Cached Hugging Face blobs
|
|
75
|
-
whose ETags encode Git or LFS checksums are re-hashed as part of verification.
|
|
36
|
+
## Fine-tune a model
|
|
76
37
|
|
|
77
|
-
|
|
78
|
-
second terminal:
|
|
38
|
+
Create a project on the GPU host:
|
|
79
39
|
|
|
80
40
|
```bash
|
|
81
|
-
|
|
82
|
-
tt-local
|
|
41
|
+
mkdir support-adapter && cd support-adapter
|
|
42
|
+
tt-local init --name "Support Adapter" --model Qwen/Qwen3.5-2B --profile spark
|
|
83
43
|
```
|
|
84
44
|
|
|
85
|
-
|
|
86
|
-
|
|
45
|
+
This creates `tunedtensor.json` and `local-runner.json`. Edit the generated
|
|
46
|
+
system prompt and replace both placeholder examples with representative,
|
|
47
|
+
different examples. Two examples are the minimum for a real run because
|
|
48
|
+
training and evaluation must not use the same row.
|
|
87
49
|
|
|
88
|
-
|
|
50
|
+
Preflight the machine, input, and cached base model:
|
|
89
51
|
|
|
90
52
|
```bash
|
|
91
|
-
tt-local
|
|
92
|
-
tt-local
|
|
93
|
-
tt-local models
|
|
53
|
+
tt-local doctor tunedtensor.json
|
|
54
|
+
tt-local validate tunedtensor.json
|
|
55
|
+
tt-local models prefetch tunedtensor.json
|
|
56
|
+
tt-local models verify-base tunedtensor.json
|
|
94
57
|
```
|
|
95
58
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
The model is registered immediately after its artifact passes verification,
|
|
100
|
-
so it remains discoverable if candidate evaluation or report generation later
|
|
101
|
-
fails.
|
|
102
|
-
|
|
103
|
-
## Serve a Trained Adapter
|
|
104
|
-
|
|
105
|
-
`tt-local serve` is the run dashboard. To serve a trained TT Local adapter as
|
|
106
|
-
an OpenAI-compatible model endpoint, use the nested model command:
|
|
107
|
-
|
|
108
|
-
```bash
|
|
109
|
-
tt-local models serve local-<run-id> \
|
|
110
|
-
--config local-runner.json \
|
|
111
|
-
--spec tunedtensor.json \
|
|
112
|
-
--host 127.0.0.1 \
|
|
113
|
-
--port 8000
|
|
114
|
-
```
|
|
59
|
+
Commands discover `local-runner.json` beside the spec. Use
|
|
60
|
+
`--config /path/to/local-runner.json` to select another config explicitly.
|
|
115
61
|
|
|
116
|
-
|
|
117
|
-
PEFT adapter through the bundled uv environment, and binds to localhost by
|
|
118
|
-
default. Test it from another terminal:
|
|
62
|
+
Run the complete base-evaluate, train, tuned-evaluate, and report workflow:
|
|
119
63
|
|
|
120
64
|
```bash
|
|
121
|
-
|
|
122
|
-
curl http://127.0.0.1:8000/v1/models
|
|
123
|
-
curl http://127.0.0.1:8000/v1/chat/completions \
|
|
124
|
-
-H 'Content-Type: application/json' \
|
|
125
|
-
-d '{"messages":[{"role":"user","content":"Hello"}]}'
|
|
65
|
+
tt-local run tunedtensor.json
|
|
126
66
|
```
|
|
127
67
|
|
|
128
|
-
|
|
129
|
-
Non-loopback binds require both `--allow-remote` and
|
|
130
|
-
`--api-key-env <environment-variable>`; requests must then send that value as
|
|
131
|
-
a Bearer token. The server accepts multimodal images only as bounded `data:`
|
|
132
|
-
URIs and does not support streaming yet.
|
|
133
|
-
|
|
134
|
-
## Cache and Evaluation Semantics
|
|
135
|
-
|
|
136
|
-
`paths.modelCache` means Hugging Face `HF_HOME` everywhere. Hub snapshots live
|
|
137
|
-
under `<modelCache>/hub`; prefetch, training, evaluation, doctor, and serving
|
|
138
|
-
all use that same layout. Set `HF_TOKEN` in the environment or project `.env`
|
|
139
|
-
before prefetching a gated model. Do not commit `.env`.
|
|
140
|
-
|
|
141
|
-
For reproducible remote loading, set
|
|
142
|
-
`hyperparameters.base_model_revision` to an immutable Hugging Face commit.
|
|
143
|
-
Alternatively, `paths.baseModel` may point to a complete local Hugging Face
|
|
144
|
-
snapshot directory. It cannot point to a model archive or standalone weights
|
|
145
|
-
file; TT Local requires non-empty model weights, `config.json`, tokenizer
|
|
146
|
-
metadata and vocabulary, and every indexed shard. Local snapshot contents are
|
|
147
|
-
fingerprinted so a byte change invalidates dependent stages.
|
|
148
|
-
|
|
149
|
-
Shared baseline-cache entries are used only when the model and every local
|
|
150
|
-
input have a stable content identity. An unpinned remote model or remotely
|
|
151
|
-
hosted image bypasses the cache rather than risking stale evaluation results.
|
|
152
|
-
|
|
153
|
-
The safe default scoring mode is deterministic `exact_match`. To use an
|
|
154
|
-
OpenRouter judge, configure `evaluation.scoring.mode: "llm_judge"`, an `llm`
|
|
155
|
-
block, and its API-key environment variable. Judge mode fails preflight when
|
|
156
|
-
those are missing. Exact-match fallback occurs only when explicitly requested
|
|
157
|
-
with `evaluation.scoring.fallback: "exact_match"`; fallback-scored results are
|
|
158
|
-
marked and never written to the judge baseline cache.
|
|
159
|
-
|
|
160
|
-
Before the first real run, you can download the configured Hugging Face base
|
|
161
|
-
model into `paths.modelCache` explicitly:
|
|
68
|
+
Inspect the evidence and verify the adapter:
|
|
162
69
|
|
|
163
70
|
```bash
|
|
164
|
-
tt-local
|
|
165
|
-
tt-local models verify
|
|
71
|
+
tt-local runs report <run-id>
|
|
72
|
+
tt-local models verify local-<run-id>
|
|
166
73
|
```
|
|
167
74
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
Individual `run --stage ...` commands finish in the terminal
|
|
172
|
-
`stage_completed` state, so `runs watch` exits instead of waiting for stages
|
|
173
|
-
that were intentionally not requested.
|
|
75
|
+
The report is evidence on the selected evaluation rows, not a guarantee of
|
|
76
|
+
general model improvement.
|
|
174
77
|
|
|
175
|
-
|
|
176
|
-
Relative dataset and image paths resolve from the behavior spec or request
|
|
177
|
-
file's directory. The published package includes `uv.lock` so bundled Python
|
|
178
|
-
dependencies are reproducible.
|
|
78
|
+
## Use a chat JSONL dataset
|
|
179
79
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
TT Local supports first-class offline DPO for text causal-LM models. Set
|
|
183
|
-
`training_method` to `dpo` and provide a prebuilt preference JSONL training
|
|
184
|
-
dataset:
|
|
80
|
+
For more than a few examples, set `dataset_prebuilt` in `tunedtensor.json`:
|
|
185
81
|
|
|
186
82
|
```json
|
|
187
83
|
{
|
|
188
|
-
"training_method": "dpo",
|
|
189
84
|
"dataset_prebuilt": {
|
|
190
|
-
"training": "
|
|
191
|
-
"
|
|
85
|
+
"training": "data/train.jsonl",
|
|
86
|
+
"validation": "data/validation.jsonl",
|
|
87
|
+
"format": "chat_jsonl"
|
|
192
88
|
}
|
|
193
89
|
}
|
|
194
90
|
```
|
|
195
91
|
|
|
196
|
-
Each
|
|
197
|
-
|
|
92
|
+
Each line is a JSON object with a `messages` array. The final message must be
|
|
93
|
+
the assistant answer; preceding messages form the prompt:
|
|
198
94
|
|
|
199
95
|
```json
|
|
200
|
-
{"
|
|
96
|
+
{"messages":[{"role":"system","content":"Return one sentiment label."},{"role":"user","content":"I loved it."},{"role":"assistant","content":"positive"}]}
|
|
201
97
|
```
|
|
202
98
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
`spec_snapshot.examples` with reference outputs for evaluation.
|
|
207
|
-
|
|
208
|
-
Command-backed workflows may use external model ids by setting
|
|
209
|
-
`spec_snapshot.base_model` to an `external:<id>` or `command:<id>` value, for
|
|
210
|
-
example `external:karpathy/nanochat`. The bundled `uv` trainer still requires a
|
|
211
|
-
supported Hugging Face base model, but command trainers receive adapter-focused
|
|
212
|
-
hyperparameters without injected LoRA/model-loader defaults. Custom
|
|
213
|
-
hyperparameter keys are passed through to `TT_HYPERPARAMETERS_PATH`.
|
|
214
|
-
Set `hyperparameters.base_model_revision` to an immutable Hugging Face commit
|
|
215
|
-
when a run must load and record an exact base-model revision.
|
|
99
|
+
TT Local masks every prompt token from the loss and preserves the complete
|
|
100
|
+
assistant answer when truncating a long example. Use a distinct validation or
|
|
101
|
+
test file for trustworthy evaluation.
|
|
216
102
|
|
|
217
|
-
##
|
|
218
|
-
|
|
219
|
-
Start another loop from an existing TT Local model by passing the stored local
|
|
220
|
-
model id:
|
|
103
|
+
## Serve the adapter
|
|
221
104
|
|
|
222
105
|
```bash
|
|
223
|
-
tt-local
|
|
106
|
+
tt-local serve local-<run-id> \
|
|
107
|
+
--spec tunedtensor.json \
|
|
108
|
+
--host 127.0.0.1 \
|
|
109
|
+
--port 8000
|
|
224
110
|
```
|
|
225
111
|
|
|
226
|
-
|
|
112
|
+
`tt-local models serve` is an equivalent nested command. Before launch, TT
|
|
113
|
+
Local re-hashes the stored artifact manifest and then loads the recorded base
|
|
114
|
+
model plus PEFT adapter with the bundled Python environment.
|
|
227
115
|
|
|
228
116
|
```bash
|
|
229
|
-
|
|
117
|
+
curl http://127.0.0.1:8000/health
|
|
118
|
+
curl http://127.0.0.1:8000/v1/models
|
|
119
|
+
curl http://127.0.0.1:8000/v1/chat/completions \
|
|
120
|
+
-H 'Content-Type: application/json' \
|
|
121
|
+
-d '{"messages":[{"role":"user","content":"Classify: I loved it."}]}'
|
|
230
122
|
```
|
|
231
123
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
baseline for the new run, and the bundled SFT/DPO trainers load it before
|
|
235
|
-
continuing LoRA training.
|
|
124
|
+
The default bind is localhost. A non-loopback bind requires `--allow-remote`
|
|
125
|
+
and `--api-key-env <environment-variable>`.
|
|
236
126
|
|
|
237
|
-
|
|
238
|
-
the child's base revision from the parent (or requires the same configured
|
|
239
|
-
local snapshot content), rejects conflicting base identity, and fingerprints
|
|
240
|
-
the parent artifact so changed bytes invalidate reuse. The explicit
|
|
241
|
-
`--parent-model-artifact` form is for caller-managed local artifacts and should
|
|
242
|
-
be used only when that provenance is understood.
|
|
127
|
+
## Reproducibility and local data
|
|
243
128
|
|
|
244
|
-
|
|
129
|
+
Set `paths.modelCache` in `local-runner.json` to a durable Hugging Face cache.
|
|
130
|
+
Prefetch records the resolved model commit; set
|
|
131
|
+
`hyperparameters.base_model_revision` to a 40-character Hugging Face commit
|
|
132
|
+
SHA when an immutable revision is required before download.
|
|
245
133
|
|
|
246
|
-
|
|
247
|
-
{
|
|
248
|
-
"training": {
|
|
249
|
-
"backend": "command",
|
|
250
|
-
"command": ["python", "train_adapter.py"],
|
|
251
|
-
"artifact": {
|
|
252
|
-
"framework": "nanochat",
|
|
253
|
-
"format": "custom-directory",
|
|
254
|
-
"entrypoint": "batch_command",
|
|
255
|
-
"servable": false
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
```
|
|
134
|
+
Run state is intentionally plain and recoverable:
|
|
260
135
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
npm run build
|
|
136
|
+
```text
|
|
137
|
+
.tt-local/
|
|
138
|
+
artifacts/users/<user-id>/specs/<spec-id>/runs/<run-id>/...
|
|
139
|
+
store/specs/<spec-id>/spec.json
|
|
140
|
+
store/runs/<run-id>/{request.json,state.json,progress.jsonl,run-report.json}
|
|
141
|
+
store/models/<model-id>/model.json
|
|
268
142
|
```
|
|
269
143
|
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
## License
|
|
144
|
+
Real training writes an `artifact-manifest.json` containing file sizes and
|
|
145
|
+
SHA-256 hashes. Dry runs exercise orchestration but do not register a model.
|
|
273
146
|
|
|
274
|
-
|
|
147
|
+
See [docs/spark.md](docs/spark.md) for the DGX Spark workflow and
|
|
148
|
+
[docs/architecture.md](docs/architecture.md) for the design boundary.
|
package/dist/artifacts.d.ts
CHANGED
|
@@ -77,16 +77,15 @@ export interface ArtifactManifest {
|
|
|
77
77
|
}
|
|
78
78
|
export interface ArtifactManifestModel {
|
|
79
79
|
artifact_kind: "file" | "directory";
|
|
80
|
-
format:
|
|
81
|
-
framework:
|
|
80
|
+
format: "tar.gz" | "huggingface-directory";
|
|
81
|
+
framework: "transformers-peft";
|
|
82
82
|
base_model: string;
|
|
83
83
|
base_model_revision?: string;
|
|
84
84
|
base_model_artifact_uri?: string;
|
|
85
85
|
base_model_fingerprint?: string;
|
|
86
|
-
parent_model_artifact?: string;
|
|
87
86
|
artifact_uri: string;
|
|
88
87
|
artifact_root: string;
|
|
89
|
-
servable:
|
|
88
|
+
servable: true;
|
|
90
89
|
files: ArtifactManifestFile[];
|
|
91
90
|
}
|
|
92
91
|
export interface ArtifactManifestVerification {
|
package/dist/artifacts.js
CHANGED
|
@@ -167,7 +167,6 @@ const MANIFEST_IGNORED_PATHS = new Set([
|
|
|
167
167
|
// records the next stage event or refreshes the canonical request copy.
|
|
168
168
|
"progress.jsonl",
|
|
169
169
|
"request.json",
|
|
170
|
-
"detached.log",
|
|
171
170
|
RUN_OWNER_FILE,
|
|
172
171
|
ARTIFACT_WORKFLOW_LOCK_FILE,
|
|
173
172
|
]);
|
|
@@ -243,6 +242,13 @@ function assertManifestFiles(value, label) {
|
|
|
243
242
|
paths.add(manifestPath);
|
|
244
243
|
}
|
|
245
244
|
}
|
|
245
|
+
function hasPeftAdapterFiles(files) {
|
|
246
|
+
const names = new Set(files
|
|
247
|
+
.filter((file) => file.size_bytes > 0)
|
|
248
|
+
.map((file) => file.path.split("/").at(-1)?.toLowerCase()));
|
|
249
|
+
return (names.has("adapter_config.json")
|
|
250
|
+
&& (names.has("adapter_model.safetensors") || names.has("adapter_model.bin")));
|
|
251
|
+
}
|
|
246
252
|
function parseArtifactManifest(value) {
|
|
247
253
|
if (!value || typeof value !== "object")
|
|
248
254
|
throw new Error("Artifact manifest must be an object.");
|
|
@@ -253,11 +259,25 @@ function parseArtifactManifest(value) {
|
|
|
253
259
|
assertManifestFiles(candidate.files, "file");
|
|
254
260
|
if (candidate.model) {
|
|
255
261
|
const model = candidate.model;
|
|
262
|
+
const allowedModelKeys = new Set([
|
|
263
|
+
"artifact_kind",
|
|
264
|
+
"format",
|
|
265
|
+
"framework",
|
|
266
|
+
"base_model",
|
|
267
|
+
"base_model_revision",
|
|
268
|
+
"base_model_artifact_uri",
|
|
269
|
+
"base_model_fingerprint",
|
|
270
|
+
"artifact_uri",
|
|
271
|
+
"artifact_root",
|
|
272
|
+
"servable",
|
|
273
|
+
"files",
|
|
274
|
+
]);
|
|
256
275
|
const hasBaseArtifact = model.base_model_artifact_uri !== undefined;
|
|
257
276
|
const hasBaseFingerprint = model.base_model_fingerprint !== undefined;
|
|
258
|
-
if ((model.
|
|
259
|
-
||
|
|
260
|
-
||
|
|
277
|
+
if (Object.keys(model).some((key) => !allowedModelKeys.has(key))
|
|
278
|
+
|| (model.artifact_kind !== "file" && model.artifact_kind !== "directory")
|
|
279
|
+
|| (model.format !== "tar.gz" && model.format !== "huggingface-directory")
|
|
280
|
+
|| model.framework !== "transformers-peft"
|
|
261
281
|
|| typeof model.base_model !== "string"
|
|
262
282
|
|| (model.base_model_revision !== undefined && typeof model.base_model_revision !== "string")
|
|
263
283
|
|| (model.base_model_artifact_uri !== undefined && typeof model.base_model_artifact_uri !== "string")
|
|
@@ -265,13 +285,17 @@ function parseArtifactManifest(value) {
|
|
|
265
285
|
|| hasBaseArtifact !== hasBaseFingerprint
|
|
266
286
|
|| typeof model.artifact_uri !== "string"
|
|
267
287
|
|| typeof model.artifact_root !== "string"
|
|
268
|
-
||
|
|
269
|
-
|| (model.parent_model_artifact !== undefined && typeof model.parent_model_artifact !== "string")
|
|
288
|
+
|| model.servable !== true
|
|
270
289
|
|| !Array.isArray(model.files)
|
|
271
|
-
|| model.files.length === 0
|
|
290
|
+
|| model.files.length === 0
|
|
291
|
+
|| (model.artifact_kind === "file" && model.format !== "tar.gz")
|
|
292
|
+
|| (model.artifact_kind === "directory" && model.format !== "huggingface-directory")) {
|
|
272
293
|
throw new Error("Artifact manifest contains invalid model contract metadata.");
|
|
273
294
|
}
|
|
274
295
|
assertManifestFiles(model.files, "model file");
|
|
296
|
+
if (model.artifact_kind === "directory" && !hasPeftAdapterFiles(model.files)) {
|
|
297
|
+
throw new Error("PEFT model manifest requires adapter weights and adapter_config.json.");
|
|
298
|
+
}
|
|
275
299
|
}
|
|
276
300
|
return candidate;
|
|
277
301
|
}
|
|
@@ -280,6 +304,12 @@ export async function writeArtifactManifest(artifacts, options = {}) {
|
|
|
280
304
|
const paths = await listArtifactFiles(artifacts.runDir);
|
|
281
305
|
let model;
|
|
282
306
|
if (options.model) {
|
|
307
|
+
if (options.model.framework !== "transformers-peft"
|
|
308
|
+
|| options.model.servable !== true
|
|
309
|
+
|| (options.model.artifact_kind === "file" && options.model.format !== "tar.gz")
|
|
310
|
+
|| (options.model.artifact_kind === "directory" && options.model.format !== "huggingface-directory")) {
|
|
311
|
+
throw new Error("TT Local model artifacts must be verified, servable Transformers PEFT adapters.");
|
|
312
|
+
}
|
|
283
313
|
if ((options.model.base_model_artifact_uri === undefined)
|
|
284
314
|
!== (options.model.base_model_fingerprint === undefined)) {
|
|
285
315
|
throw new Error("Model contracts must record a local base-model URI and fingerprint together.");
|
|
@@ -289,25 +319,26 @@ export async function writeArtifactManifest(artifacts, options = {}) {
|
|
|
289
319
|
? [portableRelative(dirname(modelRoot), modelRoot)]
|
|
290
320
|
: await listArtifactFiles(modelRoot);
|
|
291
321
|
const checksumRoot = options.model.artifact_kind === "file" ? dirname(modelRoot) : modelRoot;
|
|
292
|
-
if (options.model.artifact_kind === "file"
|
|
293
|
-
|
|
322
|
+
if (options.model.artifact_kind === "file") {
|
|
323
|
+
if (!modelRoot.toLowerCase().endsWith(".tar.gz")) {
|
|
324
|
+
throw new Error(`PEFT file artifacts must be .tar.gz archives: ${modelRoot}`);
|
|
325
|
+
}
|
|
294
326
|
const archive = await verifyTarGzipArchive(modelRoot);
|
|
295
|
-
if (
|
|
296
|
-
&& archive.adapter_weight_entries === 0) {
|
|
327
|
+
if (archive.adapter_weight_entries === 0 || archive.adapter_weight_bytes === 0) {
|
|
297
328
|
throw new Error(`PEFT model archive contains no adapter_model.safetensors or adapter_model.bin: ${modelRoot}`);
|
|
298
329
|
}
|
|
299
|
-
if (
|
|
300
|
-
&& archive.adapter_config_entries === 0) {
|
|
330
|
+
if (archive.adapter_config_entries === 0) {
|
|
301
331
|
throw new Error(`PEFT model archive contains no adapter_config.json: ${modelRoot}`);
|
|
302
332
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
333
|
+
}
|
|
334
|
+
const modelFiles = await Promise.all(modelPaths.map((path) => describeArtifactFile(checksumRoot, path)));
|
|
335
|
+
if (options.model.artifact_kind === "directory" && !hasPeftAdapterFiles(modelFiles)) {
|
|
336
|
+
throw new Error(`PEFT model directory requires adapter weights and adapter_config.json: ${modelRoot}`);
|
|
306
337
|
}
|
|
307
338
|
model = {
|
|
308
339
|
...options.model,
|
|
309
340
|
artifact_root: modelRoot,
|
|
310
|
-
files:
|
|
341
|
+
files: modelFiles,
|
|
311
342
|
};
|
|
312
343
|
}
|
|
313
344
|
const manifest = {
|
|
@@ -377,26 +408,23 @@ export async function verifyArtifactManifest(manifestPath, options = {}) {
|
|
|
377
408
|
const expectedModelPaths = new Set(manifest.model.files.map((file) => file.path));
|
|
378
409
|
unexpected.push(...modelActualPaths.filter((path) => !expectedModelPaths.has(path)).map((path) => `model:${path}`));
|
|
379
410
|
}
|
|
380
|
-
if (manifest.model.artifact_kind === "file"
|
|
381
|
-
&& (manifest.model.format === "tar.gz" || modelRoot.endsWith(".tar.gz"))) {
|
|
411
|
+
if (manifest.model.artifact_kind === "file") {
|
|
382
412
|
try {
|
|
383
413
|
const archive = await verifyTarGzipArchive(modelRoot);
|
|
384
|
-
if (
|
|
385
|
-
&& archive.adapter_weight_entries === 0) {
|
|
414
|
+
if (archive.adapter_weight_entries === 0 || archive.adapter_weight_bytes === 0) {
|
|
386
415
|
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_adapter_weights`);
|
|
387
416
|
}
|
|
388
|
-
if (
|
|
389
|
-
&& archive.adapter_config_entries === 0) {
|
|
417
|
+
if (archive.adapter_config_entries === 0) {
|
|
390
418
|
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_adapter_config`);
|
|
391
419
|
}
|
|
392
|
-
if (manifest.model.framework === "transformers-full" && archive.full_model_weight_entries === 0) {
|
|
393
|
-
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_full_model_weights`);
|
|
394
|
-
}
|
|
395
420
|
}
|
|
396
421
|
catch {
|
|
397
422
|
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:invalid_archive`);
|
|
398
423
|
}
|
|
399
424
|
}
|
|
425
|
+
else if (!hasPeftAdapterFiles(await Promise.all(modelActualPaths.map((path) => describeArtifactFile(checksumRoot, path))))) {
|
|
426
|
+
changed.push("model:missing_peft_adapter");
|
|
427
|
+
}
|
|
400
428
|
}
|
|
401
429
|
return {
|
|
402
430
|
valid: missing.length === 0 && changed.length === 0 && unexpected.length === 0,
|
|
@@ -564,13 +592,7 @@ export async function verifyTarGzipArchive(path) {
|
|
|
564
592
|
const recognizedPayload = Boolean(name)
|
|
565
593
|
&& name !== "training_args.bin"
|
|
566
594
|
&& (name.endsWith(".safetensors")
|
|
567
|
-
|| name.endsWith(".bin")
|
|
568
|
-
|| name.endsWith(".pt")
|
|
569
|
-
|| name.endsWith(".pth")
|
|
570
|
-
|| name.endsWith(".gguf")
|
|
571
|
-
|| name.endsWith(".onnx")
|
|
572
|
-
|| name.endsWith(".ckpt")
|
|
573
|
-
|| name.endsWith(".npz"));
|
|
595
|
+
|| name.endsWith(".bin"));
|
|
574
596
|
const adapterWeight = name === "adapter_model.safetensors" || name === "adapter_model.bin";
|
|
575
597
|
const fullModelWeight = Boolean(name)
|
|
576
598
|
&& ((name.endsWith(".safetensors") && !name.startsWith("adapter_"))
|