@tuned-tensor/local 0.2.8 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +84 -204
- package/dist/artifacts.d.ts +3 -4
- package/dist/artifacts.js +55 -33
- package/dist/artifacts.js.map +1 -1
- package/dist/compare.d.ts +1 -8
- package/dist/compare.js +1 -23
- package/dist/compare.js.map +1 -1
- package/dist/contracts.d.ts +59 -366
- package/dist/contracts.js +73 -224
- package/dist/contracts.js.map +1 -1
- package/dist/dataset.d.ts +2 -4
- package/dist/dataset.js +31 -125
- package/dist/dataset.js.map +1 -1
- package/dist/doctor.d.ts +2 -3
- package/dist/doctor.js +23 -161
- package/dist/doctor.js.map +1 -1
- package/dist/evaluation.d.ts +11 -71
- package/dist/evaluation.js +212 -572
- package/dist/evaluation.js.map +1 -1
- package/dist/huggingface-cache.d.ts +8 -7
- package/dist/huggingface-cache.js +16 -8
- package/dist/huggingface-cache.js.map +1 -1
- package/dist/index.d.ts +0 -10
- package/dist/index.js +205 -628
- package/dist/index.js.map +1 -1
- package/dist/local-project.d.ts +1 -11
- package/dist/local-project.js +33 -61
- package/dist/local-project.js.map +1 -1
- package/dist/model-registry.d.ts +3 -16
- package/dist/model-registry.js +76 -292
- package/dist/model-registry.js.map +1 -1
- package/dist/model-server.d.ts +1 -2
- package/dist/model-server.js +9 -18
- package/dist/model-server.js.map +1 -1
- package/dist/orchestrator.d.ts +11 -25
- package/dist/orchestrator.js +246 -566
- package/dist/orchestrator.js.map +1 -1
- package/dist/prefetch.d.ts +1 -5
- package/dist/prefetch.js +14 -19
- package/dist/prefetch.js.map +1 -1
- package/dist/process-runner.d.ts +10 -29
- package/dist/process-runner.js +63 -169
- package/dist/process-runner.js.map +1 -1
- package/dist/process-training.d.ts +0 -1
- package/dist/process-training.js +10 -87
- package/dist/process-training.js.map +1 -1
- package/dist/store.d.ts +1 -3
- package/dist/store.js +92 -290
- package/dist/store.js.map +1 -1
- package/docs/architecture.md +87 -152
- package/docs/spark.md +66 -97
- package/examples/dry-runner.json +14 -0
- package/examples/local-runner.json +8 -6
- package/examples/smoke-spec.json +41 -0
- package/package.json +6 -6
- package/training/local-runner/pyproject.toml +0 -5
- package/training/local-runner/src/evaluate.py +89 -234
- package/training/local-runner/src/model_contract.py +47 -0
- package/training/local-runner/src/prefetch.py +18 -4
- package/training/local-runner/src/serve.py +54 -115
- package/training/local-runner/src/sft_data.py +97 -0
- package/training/local-runner/src/train.py +146 -346
- package/training/local-runner/uv.lock +0 -1197
- package/dist/labeling-sanitize.d.ts +0 -31
- package/dist/labeling-sanitize.js +0 -158
- package/dist/labeling-sanitize.js.map +0 -1
- package/dist/labeling.d.ts +0 -155
- package/dist/labeling.js +0 -496
- package/dist/labeling.js.map +0 -1
- package/dist/openrouter.d.ts +0 -29
- package/dist/openrouter.js +0 -66
- package/dist/openrouter.js.map +0 -1
- package/dist/server.d.ts +0 -9
- package/dist/server.js +0 -211
- package/dist/server.js.map +0 -1
- package/docs/local-workflow-remediation-2026-07-13.md +0 -130
- package/docs/local-workflow-ux-review-2026-07-13.md +0 -458
- package/examples/dpo-preferences.jsonl +0 -2
- package/examples/dpo-run-request.json +0 -34
- package/examples/smoke-run-request.json +0 -34
- package/training/local-runner/src/train_dpo.py +0 -286
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,73 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to TT Local will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## Unreleased
|
|
6
|
+
|
|
7
|
+
## 0.3.0 - 2026-07-26
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- Added one certified end-to-end local workflow for CUDA LoRA SFT of
|
|
12
|
+
`Qwen/Qwen3.5-2B`, including immutable snapshot prefetch, held-out
|
|
13
|
+
base-versus-tuned evaluation, verified PEFT artifacts, and
|
|
14
|
+
OpenAI-compatible local serving.
|
|
15
|
+
- Added exact Qwen model-contract checks in both TypeScript and Python, plus
|
|
16
|
+
strict text-only chat JSONL validation and assistant-only SFT data tests.
|
|
17
|
+
|
|
18
|
+
### Removed
|
|
19
|
+
|
|
20
|
+
- **Breaking:** reduced TT Local to its end-to-end verified purpose: text LoRA
|
|
21
|
+
SFT of `Qwen/Qwen3.5-2B` on CUDA. Removed DPO, multimodal and continuation
|
|
22
|
+
inputs, arbitrary training/evaluation commands, external model contracts,
|
|
23
|
+
cloud teacher labeling and LLM judging, the classic-ML Study subsystem, and
|
|
24
|
+
the local dashboard.
|
|
25
|
+
- Removed the SQLite metadata mirror. Canonical JSON/JSONL files are now the
|
|
26
|
+
only state store and remain directly inspectable and recoverable.
|
|
27
|
+
- Removed implicit working-directory `.env` loading. Credentials and serving
|
|
28
|
+
keys must be provided explicitly through the process environment.
|
|
29
|
+
- Removed the incomplete detached-run, watch, and remote-cancel surfaces.
|
|
30
|
+
Workflows now run in the foreground so process and state lifecycles cannot
|
|
31
|
+
diverge after a host restart or worker crash.
|
|
32
|
+
|
|
33
|
+
### Changed
|
|
34
|
+
|
|
35
|
+
- Training now computes loss only on assistant answer tokens and truncates the
|
|
36
|
+
prompt before the answer. CUDA training and tokenizer chat-template failures
|
|
37
|
+
fail fast instead of silently changing behavior.
|
|
38
|
+
- Runner, spec, and hyperparameter schemas are strict; obsolete and misspelled
|
|
39
|
+
fields are rejected rather than ignored.
|
|
40
|
+
- Training, evaluation, and serving now always use the bundled locked Python
|
|
41
|
+
project. Removed public runner project, working-directory, provider, and
|
|
42
|
+
environment overrides.
|
|
43
|
+
- Bundled `uv` commands now use `--frozen` and a content-keyed virtual
|
|
44
|
+
environment in the writable user cache instead of modifying the npm package.
|
|
45
|
+
- After prefetch verifies an immutable snapshot, training, evaluation, and
|
|
46
|
+
serving enforce offline Hugging Face cache use and the current Transformers
|
|
47
|
+
`dtype` loading API.
|
|
48
|
+
- Explicit base-model revisions must be immutable 40-character Hugging Face
|
|
49
|
+
commit SHAs; moving branches and tags are rejected.
|
|
50
|
+
- Default holdout sampling is stable across runs of the same behavior spec,
|
|
51
|
+
and baseline cache identity no longer includes the per-run UUID.
|
|
52
|
+
- The supported workflow is now documented and tested as one coherent path:
|
|
53
|
+
initialize, preflight, prepare held-out data, train, compare, verify, and
|
|
54
|
+
serve a local PEFT adapter.
|
|
55
|
+
- Prebuilt datasets with both validation and test files now use validation for
|
|
56
|
+
normal iterative evaluation, preserving test as a separate holdout.
|
|
57
|
+
|
|
58
|
+
### Fixed
|
|
59
|
+
|
|
60
|
+
- Completed process-group teardown when a logged command's direct child exits,
|
|
61
|
+
including an opt-in cleanup mode for successful commands that must not leave
|
|
62
|
+
background descendants running.
|
|
63
|
+
|
|
64
|
+
## 0.2.9 - 2026-07-14
|
|
65
|
+
|
|
66
|
+
### Changed
|
|
67
|
+
|
|
68
|
+
- Reframed TT Local as local-first fine-tuning with paired baseline-vs-tuned
|
|
69
|
+
evaluation for small open-weight models, and aligned the README, package
|
|
70
|
+
metadata, and CLI help with that positioning.
|
|
71
|
+
|
|
5
72
|
## 0.2.8 - 2026-07-14
|
|
6
73
|
|
|
7
74
|
### Added
|
package/README.md
CHANGED
|
@@ -3,266 +3,146 @@
|
|
|
3
3
|
[](https://github.com/tunedtensor/tuned-tensor-local/actions/workflows/ci.yml)
|
|
4
4
|
[](https://www.npmjs.com/package/@tuned-tensor/local)
|
|
5
5
|
|
|
6
|
-
TT Local
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
TT Local fine-tunes an open-weight language model on your own NVIDIA GPU. It
|
|
7
|
+
turns a small behavior spec or chat JSONL dataset into a verified LoRA adapter,
|
|
8
|
+
compares the base and tuned model on held-out examples, and can serve the
|
|
9
|
+
adapter through an OpenAI-compatible endpoint.
|
|
9
10
|
|
|
10
|
-
|
|
11
|
+
The current release deliberately has one supported path:
|
|
11
12
|
|
|
12
|
-
|
|
13
|
+
- text supervised fine-tuning (SFT);
|
|
14
|
+
- `Qwen/Qwen3.5-2B`;
|
|
15
|
+
- LoRA/PEFT on CUDA;
|
|
16
|
+
- deterministic exact-match or JSON-field evaluation;
|
|
17
|
+
- local files for datasets, state, reports, and model artifacts.
|
|
13
18
|
|
|
14
|
-
|
|
19
|
+
New model families and training methods should be added only after they have a
|
|
20
|
+
repeatable end-to-end GPU acceptance test.
|
|
15
21
|
|
|
16
|
-
|
|
17
|
-
npm install -g @tuned-tensor/local
|
|
18
|
-
tt-local info
|
|
19
|
-
```
|
|
20
|
-
|
|
21
|
-
The bundled SFT, DPO, and Transformers evaluator path also needs `uv`:
|
|
22
|
-
|
|
23
|
-
```bash
|
|
24
|
-
uv --version
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
Custom training or evaluation workflows can use command entrypoints instead of
|
|
28
|
-
`uv`.
|
|
22
|
+
## Requirements
|
|
29
23
|
|
|
30
|
-
|
|
31
|
-
`
|
|
32
|
-
|
|
24
|
+
- Node.js 22 or later
|
|
25
|
+
- [`uv`](https://docs.astral.sh/uv/)
|
|
26
|
+
- an NVIDIA CUDA host with enough free disk space for the model cache and run
|
|
27
|
+
artifacts
|
|
33
28
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
On an NVIDIA Spark or another CUDA host, initialize both the behavior spec and
|
|
37
|
-
a durable project-local runner config:
|
|
29
|
+
The npm package includes the locked Python training environment.
|
|
38
30
|
|
|
39
31
|
```bash
|
|
40
|
-
|
|
41
|
-
tt-local
|
|
32
|
+
npm install -g @tuned-tensor/local
|
|
33
|
+
tt-local info
|
|
42
34
|
```
|
|
43
35
|
|
|
44
|
-
|
|
45
|
-
TT Local refuses to validate or train the unchanged placeholder so an
|
|
46
|
-
accidental run cannot spend GPU time on it.
|
|
36
|
+
## Fine-tune a model
|
|
47
37
|
|
|
48
|
-
|
|
38
|
+
Create a project on the GPU host:
|
|
49
39
|
|
|
50
40
|
```bash
|
|
51
|
-
|
|
52
|
-
tt-local
|
|
41
|
+
mkdir support-adapter && cd support-adapter
|
|
42
|
+
tt-local init --name "Support Adapter" --model Qwen/Qwen3.5-2B --profile spark
|
|
53
43
|
```
|
|
54
44
|
|
|
55
|
-
`
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
45
|
+
This creates `tunedtensor.json` and `local-runner.json`. Edit the generated
|
|
46
|
+
system prompt and replace both placeholder examples with representative,
|
|
47
|
+
different examples. Two examples are the minimum for a real run because
|
|
48
|
+
training and evaluation must not use the same row.
|
|
59
49
|
|
|
60
|
-
|
|
50
|
+
Preflight the machine, input, and cached base model:
|
|
61
51
|
|
|
62
52
|
```bash
|
|
63
|
-
tt-local
|
|
64
|
-
tt-local
|
|
65
|
-
tt-local
|
|
53
|
+
tt-local doctor tunedtensor.json
|
|
54
|
+
tt-local validate tunedtensor.json
|
|
55
|
+
tt-local models prefetch tunedtensor.json
|
|
56
|
+
tt-local models verify-base tunedtensor.json
|
|
66
57
|
```
|
|
67
58
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
the cached snapshot structure, indexed weight shards, tokenizer files, and
|
|
71
|
-
non-empty weights without downloading a replacement. Cached Hugging Face blobs
|
|
72
|
-
whose ETags encode Git or LFS checksums are re-hashed as part of verification.
|
|
59
|
+
Commands discover `local-runner.json` beside the spec. Use
|
|
60
|
+
`--config /path/to/local-runner.json` to select another config explicitly.
|
|
73
61
|
|
|
74
|
-
|
|
75
|
-
second terminal:
|
|
62
|
+
Run the complete base-evaluate, train, tuned-evaluate, and report workflow:
|
|
76
63
|
|
|
77
64
|
```bash
|
|
78
|
-
tt-local run tunedtensor.json
|
|
79
|
-
tt-local runs watch <run-id> --config local-runner.json
|
|
65
|
+
tt-local run tunedtensor.json
|
|
80
66
|
```
|
|
81
67
|
|
|
82
|
-
|
|
83
|
-
copy-paste watch/cancel commands before model work begins.
|
|
84
|
-
|
|
85
|
-
After training:
|
|
68
|
+
Inspect the evidence and verify the adapter:
|
|
86
69
|
|
|
87
70
|
```bash
|
|
88
|
-
tt-local runs
|
|
89
|
-
tt-local
|
|
90
|
-
tt-local models verify local-<run-id> --config local-runner.json
|
|
71
|
+
tt-local runs report <run-id>
|
|
72
|
+
tt-local models verify local-<run-id>
|
|
91
73
|
```
|
|
92
74
|
|
|
93
|
-
The
|
|
94
|
-
|
|
95
|
-
fails.
|
|
96
|
-
|
|
97
|
-
## Serve a Trained Adapter
|
|
75
|
+
The report is evidence on the selected evaluation rows, not a guarantee of
|
|
76
|
+
general model improvement.
|
|
98
77
|
|
|
99
|
-
|
|
100
|
-
an OpenAI-compatible model endpoint, use the nested model command:
|
|
78
|
+
## Use a chat JSONL dataset
|
|
101
79
|
|
|
102
|
-
|
|
103
|
-
tt-local models serve local-<run-id> \
|
|
104
|
-
--config local-runner.json \
|
|
105
|
-
--spec tunedtensor.json \
|
|
106
|
-
--host 127.0.0.1 \
|
|
107
|
-
--port 8000
|
|
108
|
-
```
|
|
109
|
-
|
|
110
|
-
The server verifies the stored artifact, loads the recorded base model and
|
|
111
|
-
PEFT adapter through the bundled uv environment, and binds to localhost by
|
|
112
|
-
default. Test it from another terminal:
|
|
113
|
-
|
|
114
|
-
```bash
|
|
115
|
-
curl http://127.0.0.1:8000/health
|
|
116
|
-
curl http://127.0.0.1:8000/v1/models
|
|
117
|
-
curl http://127.0.0.1:8000/v1/chat/completions \
|
|
118
|
-
-H 'Content-Type: application/json' \
|
|
119
|
-
-d '{"messages":[{"role":"user","content":"Hello"}]}'
|
|
120
|
-
```
|
|
121
|
-
|
|
122
|
-
Use `--device cuda|mps|cpu|auto` to override the configured inference device.
|
|
123
|
-
Non-loopback binds require both `--allow-remote` and
|
|
124
|
-
`--api-key-env <environment-variable>`; requests must then send that value as
|
|
125
|
-
a Bearer token. The server accepts multimodal images only as bounded `data:`
|
|
126
|
-
URIs and does not support streaming yet.
|
|
127
|
-
|
|
128
|
-
## Cache and Evaluation Semantics
|
|
129
|
-
|
|
130
|
-
`paths.modelCache` means Hugging Face `HF_HOME` everywhere. Hub snapshots live
|
|
131
|
-
under `<modelCache>/hub`; prefetch, training, evaluation, doctor, and serving
|
|
132
|
-
all use that same layout. Set `HF_TOKEN` in the environment or project `.env`
|
|
133
|
-
before prefetching a gated model. Do not commit `.env`.
|
|
134
|
-
|
|
135
|
-
For reproducible remote loading, set
|
|
136
|
-
`hyperparameters.base_model_revision` to an immutable Hugging Face commit.
|
|
137
|
-
Alternatively, `paths.baseModel` may point to a complete local Hugging Face
|
|
138
|
-
snapshot directory. It cannot point to a model archive or standalone weights
|
|
139
|
-
file; TT Local requires non-empty model weights, `config.json`, tokenizer
|
|
140
|
-
metadata and vocabulary, and every indexed shard. Local snapshot contents are
|
|
141
|
-
fingerprinted so a byte change invalidates dependent stages.
|
|
142
|
-
|
|
143
|
-
Shared baseline-cache entries are used only when the model and every local
|
|
144
|
-
input have a stable content identity. An unpinned remote model or remotely
|
|
145
|
-
hosted image bypasses the cache rather than risking stale evaluation results.
|
|
146
|
-
|
|
147
|
-
The safe default scoring mode is deterministic `exact_match`. To use an
|
|
148
|
-
OpenRouter judge, configure `evaluation.scoring.mode: "llm_judge"`, an `llm`
|
|
149
|
-
block, and its API-key environment variable. Judge mode fails preflight when
|
|
150
|
-
those are missing. Exact-match fallback occurs only when explicitly requested
|
|
151
|
-
with `evaluation.scoring.fallback: "exact_match"`; fallback-scored results are
|
|
152
|
-
marked and never written to the judge baseline cache.
|
|
153
|
-
|
|
154
|
-
Before the first real run, you can download the configured Hugging Face base
|
|
155
|
-
model into `paths.modelCache` explicitly:
|
|
156
|
-
|
|
157
|
-
```bash
|
|
158
|
-
tt-local models prefetch tunedtensor.json --config local-runner.json
|
|
159
|
-
tt-local models verify-base tunedtensor.json --config local-runner.json
|
|
160
|
-
```
|
|
161
|
-
|
|
162
|
-
Without this step, the first non-dry `tt-local run` downloads the base model
|
|
163
|
-
when baseline evaluation or training first loads it.
|
|
164
|
-
|
|
165
|
-
Individual `run --stage ...` commands finish in the terminal
|
|
166
|
-
`stage_completed` state, so `runs watch` exits instead of waiting for stages
|
|
167
|
-
that were intentionally not requested.
|
|
168
|
-
|
|
169
|
-
Relative paths in a runner config resolve from the config file's directory.
|
|
170
|
-
Relative dataset and image paths resolve from the behavior spec or request
|
|
171
|
-
file's directory. The published package includes `uv.lock` so bundled Python
|
|
172
|
-
dependencies are reproducible.
|
|
173
|
-
|
|
174
|
-
## DPO
|
|
175
|
-
|
|
176
|
-
TT Local supports first-class offline DPO for text causal-LM models. Set
|
|
177
|
-
`training_method` to `dpo` and provide a prebuilt preference JSONL training
|
|
178
|
-
dataset:
|
|
80
|
+
For more than a few examples, set `dataset_prebuilt` in `tunedtensor.json`:
|
|
179
81
|
|
|
180
82
|
```json
|
|
181
83
|
{
|
|
182
|
-
"training_method": "dpo",
|
|
183
84
|
"dataset_prebuilt": {
|
|
184
|
-
"training": "
|
|
185
|
-
"
|
|
85
|
+
"training": "data/train.jsonl",
|
|
86
|
+
"validation": "data/validation.jsonl",
|
|
87
|
+
"format": "chat_jsonl"
|
|
186
88
|
}
|
|
187
89
|
}
|
|
188
90
|
```
|
|
189
91
|
|
|
190
|
-
Each
|
|
191
|
-
|
|
92
|
+
Each line is a JSON object with a `messages` array. The final message must be
|
|
93
|
+
the assistant answer; preceding messages form the prompt:
|
|
192
94
|
|
|
193
95
|
```json
|
|
194
|
-
{"
|
|
96
|
+
{"messages":[{"role":"system","content":"Return one sentiment label."},{"role":"user","content":"I loved it."},{"role":"assistant","content":"positive"}]}
|
|
195
97
|
```
|
|
196
98
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
`spec_snapshot.examples` with reference outputs for evaluation.
|
|
201
|
-
|
|
202
|
-
Command-backed workflows may use external model ids by setting
|
|
203
|
-
`spec_snapshot.base_model` to an `external:<id>` or `command:<id>` value, for
|
|
204
|
-
example `external:karpathy/nanochat`. The bundled `uv` trainer still requires a
|
|
205
|
-
supported Hugging Face base model, but command trainers receive adapter-focused
|
|
206
|
-
hyperparameters without injected LoRA/model-loader defaults. Custom
|
|
207
|
-
hyperparameter keys are passed through to `TT_HYPERPARAMETERS_PATH`.
|
|
208
|
-
Set `hyperparameters.base_model_revision` to an immutable Hugging Face commit
|
|
209
|
-
when a run must load and record an exact base-model revision.
|
|
99
|
+
TT Local masks every prompt token from the loss and preserves the complete
|
|
100
|
+
assistant answer when truncating a long example. Use a distinct validation or
|
|
101
|
+
test file for trustworthy evaluation.
|
|
210
102
|
|
|
211
|
-
##
|
|
212
|
-
|
|
213
|
-
Start another loop from an existing TT Local model by passing the stored local
|
|
214
|
-
model id:
|
|
103
|
+
## Serve the adapter
|
|
215
104
|
|
|
216
105
|
```bash
|
|
217
|
-
tt-local
|
|
106
|
+
tt-local serve local-<run-id> \
|
|
107
|
+
--spec tunedtensor.json \
|
|
108
|
+
--host 127.0.0.1 \
|
|
109
|
+
--port 8000
|
|
218
110
|
```
|
|
219
111
|
|
|
220
|
-
|
|
112
|
+
`tt-local models serve` is an equivalent nested command. Before launch, TT
|
|
113
|
+
Local re-hashes the stored artifact manifest and then loads the recorded base
|
|
114
|
+
model plus PEFT adapter with the bundled Python environment.
|
|
221
115
|
|
|
222
116
|
```bash
|
|
223
|
-
|
|
117
|
+
curl http://127.0.0.1:8000/health
|
|
118
|
+
curl http://127.0.0.1:8000/v1/models
|
|
119
|
+
curl http://127.0.0.1:8000/v1/chat/completions \
|
|
120
|
+
-H 'Content-Type: application/json' \
|
|
121
|
+
-d '{"messages":[{"role":"user","content":"Classify: I loved it."}]}'
|
|
224
122
|
```
|
|
225
123
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
baseline for the new run, and the bundled SFT/DPO trainers load it before
|
|
229
|
-
continuing LoRA training.
|
|
124
|
+
The default bind is localhost. A non-loopback bind requires `--allow-remote`
|
|
125
|
+
and `--api-key-env <environment-variable>`.
|
|
230
126
|
|
|
231
|
-
|
|
232
|
-
the child's base revision from the parent (or requires the same configured
|
|
233
|
-
local snapshot content), rejects conflicting base identity, and fingerprints
|
|
234
|
-
the parent artifact so changed bytes invalidate reuse. The explicit
|
|
235
|
-
`--parent-model-artifact` form is for caller-managed local artifacts and should
|
|
236
|
-
be used only when that provenance is understood.
|
|
127
|
+
## Reproducibility and local data
|
|
237
128
|
|
|
238
|
-
|
|
129
|
+
Set `paths.modelCache` in `local-runner.json` to a durable Hugging Face cache.
|
|
130
|
+
Prefetch records the resolved model commit; set
|
|
131
|
+
`hyperparameters.base_model_revision` to a 40-character Hugging Face commit
|
|
132
|
+
SHA when an immutable revision is required before download.
|
|
239
133
|
|
|
240
|
-
|
|
241
|
-
{
|
|
242
|
-
"training": {
|
|
243
|
-
"backend": "command",
|
|
244
|
-
"command": ["python", "train_adapter.py"],
|
|
245
|
-
"artifact": {
|
|
246
|
-
"framework": "nanochat",
|
|
247
|
-
"format": "custom-directory",
|
|
248
|
-
"entrypoint": "batch_command",
|
|
249
|
-
"servable": false
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
```
|
|
134
|
+
Run state is intentionally plain and recoverable:
|
|
254
135
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
npm run build
|
|
136
|
+
```text
|
|
137
|
+
.tt-local/
|
|
138
|
+
artifacts/users/<user-id>/specs/<spec-id>/runs/<run-id>/...
|
|
139
|
+
store/specs/<spec-id>/spec.json
|
|
140
|
+
store/runs/<run-id>/{request.json,state.json,progress.jsonl,run-report.json}
|
|
141
|
+
store/models/<model-id>/model.json
|
|
262
142
|
```
|
|
263
143
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
## License
|
|
144
|
+
Real training writes an `artifact-manifest.json` containing file sizes and
|
|
145
|
+
SHA-256 hashes. Dry runs exercise orchestration but do not register a model.
|
|
267
146
|
|
|
268
|
-
|
|
147
|
+
See [docs/spark.md](docs/spark.md) for the DGX Spark workflow and
|
|
148
|
+
[docs/architecture.md](docs/architecture.md) for the design boundary.
|
package/dist/artifacts.d.ts
CHANGED
|
@@ -77,16 +77,15 @@ export interface ArtifactManifest {
|
|
|
77
77
|
}
|
|
78
78
|
export interface ArtifactManifestModel {
|
|
79
79
|
artifact_kind: "file" | "directory";
|
|
80
|
-
format:
|
|
81
|
-
framework:
|
|
80
|
+
format: "tar.gz" | "huggingface-directory";
|
|
81
|
+
framework: "transformers-peft";
|
|
82
82
|
base_model: string;
|
|
83
83
|
base_model_revision?: string;
|
|
84
84
|
base_model_artifact_uri?: string;
|
|
85
85
|
base_model_fingerprint?: string;
|
|
86
|
-
parent_model_artifact?: string;
|
|
87
86
|
artifact_uri: string;
|
|
88
87
|
artifact_root: string;
|
|
89
|
-
servable:
|
|
88
|
+
servable: true;
|
|
90
89
|
files: ArtifactManifestFile[];
|
|
91
90
|
}
|
|
92
91
|
export interface ArtifactManifestVerification {
|
package/dist/artifacts.js
CHANGED
|
@@ -167,7 +167,6 @@ const MANIFEST_IGNORED_PATHS = new Set([
|
|
|
167
167
|
// records the next stage event or refreshes the canonical request copy.
|
|
168
168
|
"progress.jsonl",
|
|
169
169
|
"request.json",
|
|
170
|
-
"detached.log",
|
|
171
170
|
RUN_OWNER_FILE,
|
|
172
171
|
ARTIFACT_WORKFLOW_LOCK_FILE,
|
|
173
172
|
]);
|
|
@@ -243,6 +242,13 @@ function assertManifestFiles(value, label) {
|
|
|
243
242
|
paths.add(manifestPath);
|
|
244
243
|
}
|
|
245
244
|
}
|
|
245
|
+
function hasPeftAdapterFiles(files) {
|
|
246
|
+
const names = new Set(files
|
|
247
|
+
.filter((file) => file.size_bytes > 0)
|
|
248
|
+
.map((file) => file.path.split("/").at(-1)?.toLowerCase()));
|
|
249
|
+
return (names.has("adapter_config.json")
|
|
250
|
+
&& (names.has("adapter_model.safetensors") || names.has("adapter_model.bin")));
|
|
251
|
+
}
|
|
246
252
|
function parseArtifactManifest(value) {
|
|
247
253
|
if (!value || typeof value !== "object")
|
|
248
254
|
throw new Error("Artifact manifest must be an object.");
|
|
@@ -253,11 +259,25 @@ function parseArtifactManifest(value) {
|
|
|
253
259
|
assertManifestFiles(candidate.files, "file");
|
|
254
260
|
if (candidate.model) {
|
|
255
261
|
const model = candidate.model;
|
|
262
|
+
const allowedModelKeys = new Set([
|
|
263
|
+
"artifact_kind",
|
|
264
|
+
"format",
|
|
265
|
+
"framework",
|
|
266
|
+
"base_model",
|
|
267
|
+
"base_model_revision",
|
|
268
|
+
"base_model_artifact_uri",
|
|
269
|
+
"base_model_fingerprint",
|
|
270
|
+
"artifact_uri",
|
|
271
|
+
"artifact_root",
|
|
272
|
+
"servable",
|
|
273
|
+
"files",
|
|
274
|
+
]);
|
|
256
275
|
const hasBaseArtifact = model.base_model_artifact_uri !== undefined;
|
|
257
276
|
const hasBaseFingerprint = model.base_model_fingerprint !== undefined;
|
|
258
|
-
if ((model.
|
|
259
|
-
||
|
|
260
|
-
||
|
|
277
|
+
if (Object.keys(model).some((key) => !allowedModelKeys.has(key))
|
|
278
|
+
|| (model.artifact_kind !== "file" && model.artifact_kind !== "directory")
|
|
279
|
+
|| (model.format !== "tar.gz" && model.format !== "huggingface-directory")
|
|
280
|
+
|| model.framework !== "transformers-peft"
|
|
261
281
|
|| typeof model.base_model !== "string"
|
|
262
282
|
|| (model.base_model_revision !== undefined && typeof model.base_model_revision !== "string")
|
|
263
283
|
|| (model.base_model_artifact_uri !== undefined && typeof model.base_model_artifact_uri !== "string")
|
|
@@ -265,13 +285,17 @@ function parseArtifactManifest(value) {
|
|
|
265
285
|
|| hasBaseArtifact !== hasBaseFingerprint
|
|
266
286
|
|| typeof model.artifact_uri !== "string"
|
|
267
287
|
|| typeof model.artifact_root !== "string"
|
|
268
|
-
||
|
|
269
|
-
|| (model.parent_model_artifact !== undefined && typeof model.parent_model_artifact !== "string")
|
|
288
|
+
|| model.servable !== true
|
|
270
289
|
|| !Array.isArray(model.files)
|
|
271
|
-
|| model.files.length === 0
|
|
290
|
+
|| model.files.length === 0
|
|
291
|
+
|| (model.artifact_kind === "file" && model.format !== "tar.gz")
|
|
292
|
+
|| (model.artifact_kind === "directory" && model.format !== "huggingface-directory")) {
|
|
272
293
|
throw new Error("Artifact manifest contains invalid model contract metadata.");
|
|
273
294
|
}
|
|
274
295
|
assertManifestFiles(model.files, "model file");
|
|
296
|
+
if (model.artifact_kind === "directory" && !hasPeftAdapterFiles(model.files)) {
|
|
297
|
+
throw new Error("PEFT model manifest requires adapter weights and adapter_config.json.");
|
|
298
|
+
}
|
|
275
299
|
}
|
|
276
300
|
return candidate;
|
|
277
301
|
}
|
|
@@ -280,6 +304,12 @@ export async function writeArtifactManifest(artifacts, options = {}) {
|
|
|
280
304
|
const paths = await listArtifactFiles(artifacts.runDir);
|
|
281
305
|
let model;
|
|
282
306
|
if (options.model) {
|
|
307
|
+
if (options.model.framework !== "transformers-peft"
|
|
308
|
+
|| options.model.servable !== true
|
|
309
|
+
|| (options.model.artifact_kind === "file" && options.model.format !== "tar.gz")
|
|
310
|
+
|| (options.model.artifact_kind === "directory" && options.model.format !== "huggingface-directory")) {
|
|
311
|
+
throw new Error("TT Local model artifacts must be verified, servable Transformers PEFT adapters.");
|
|
312
|
+
}
|
|
283
313
|
if ((options.model.base_model_artifact_uri === undefined)
|
|
284
314
|
!== (options.model.base_model_fingerprint === undefined)) {
|
|
285
315
|
throw new Error("Model contracts must record a local base-model URI and fingerprint together.");
|
|
@@ -289,25 +319,26 @@ export async function writeArtifactManifest(artifacts, options = {}) {
|
|
|
289
319
|
? [portableRelative(dirname(modelRoot), modelRoot)]
|
|
290
320
|
: await listArtifactFiles(modelRoot);
|
|
291
321
|
const checksumRoot = options.model.artifact_kind === "file" ? dirname(modelRoot) : modelRoot;
|
|
292
|
-
if (options.model.artifact_kind === "file"
|
|
293
|
-
|
|
322
|
+
if (options.model.artifact_kind === "file") {
|
|
323
|
+
if (!modelRoot.toLowerCase().endsWith(".tar.gz")) {
|
|
324
|
+
throw new Error(`PEFT file artifacts must be .tar.gz archives: ${modelRoot}`);
|
|
325
|
+
}
|
|
294
326
|
const archive = await verifyTarGzipArchive(modelRoot);
|
|
295
|
-
if (
|
|
296
|
-
&& archive.adapter_weight_entries === 0) {
|
|
327
|
+
if (archive.adapter_weight_entries === 0 || archive.adapter_weight_bytes === 0) {
|
|
297
328
|
throw new Error(`PEFT model archive contains no adapter_model.safetensors or adapter_model.bin: ${modelRoot}`);
|
|
298
329
|
}
|
|
299
|
-
if (
|
|
300
|
-
&& archive.adapter_config_entries === 0) {
|
|
330
|
+
if (archive.adapter_config_entries === 0) {
|
|
301
331
|
throw new Error(`PEFT model archive contains no adapter_config.json: ${modelRoot}`);
|
|
302
332
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
333
|
+
}
|
|
334
|
+
const modelFiles = await Promise.all(modelPaths.map((path) => describeArtifactFile(checksumRoot, path)));
|
|
335
|
+
if (options.model.artifact_kind === "directory" && !hasPeftAdapterFiles(modelFiles)) {
|
|
336
|
+
throw new Error(`PEFT model directory requires adapter weights and adapter_config.json: ${modelRoot}`);
|
|
306
337
|
}
|
|
307
338
|
model = {
|
|
308
339
|
...options.model,
|
|
309
340
|
artifact_root: modelRoot,
|
|
310
|
-
files:
|
|
341
|
+
files: modelFiles,
|
|
311
342
|
};
|
|
312
343
|
}
|
|
313
344
|
const manifest = {
|
|
@@ -377,26 +408,23 @@ export async function verifyArtifactManifest(manifestPath, options = {}) {
|
|
|
377
408
|
const expectedModelPaths = new Set(manifest.model.files.map((file) => file.path));
|
|
378
409
|
unexpected.push(...modelActualPaths.filter((path) => !expectedModelPaths.has(path)).map((path) => `model:${path}`));
|
|
379
410
|
}
|
|
380
|
-
if (manifest.model.artifact_kind === "file"
|
|
381
|
-
&& (manifest.model.format === "tar.gz" || modelRoot.endsWith(".tar.gz"))) {
|
|
411
|
+
if (manifest.model.artifact_kind === "file") {
|
|
382
412
|
try {
|
|
383
413
|
const archive = await verifyTarGzipArchive(modelRoot);
|
|
384
|
-
if (
|
|
385
|
-
&& archive.adapter_weight_entries === 0) {
|
|
414
|
+
if (archive.adapter_weight_entries === 0 || archive.adapter_weight_bytes === 0) {
|
|
386
415
|
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_adapter_weights`);
|
|
387
416
|
}
|
|
388
|
-
if (
|
|
389
|
-
&& archive.adapter_config_entries === 0) {
|
|
417
|
+
if (archive.adapter_config_entries === 0) {
|
|
390
418
|
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_adapter_config`);
|
|
391
419
|
}
|
|
392
|
-
if (manifest.model.framework === "transformers-full" && archive.full_model_weight_entries === 0) {
|
|
393
|
-
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_full_model_weights`);
|
|
394
|
-
}
|
|
395
420
|
}
|
|
396
421
|
catch {
|
|
397
422
|
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:invalid_archive`);
|
|
398
423
|
}
|
|
399
424
|
}
|
|
425
|
+
else if (!hasPeftAdapterFiles(await Promise.all(modelActualPaths.map((path) => describeArtifactFile(checksumRoot, path))))) {
|
|
426
|
+
changed.push("model:missing_peft_adapter");
|
|
427
|
+
}
|
|
400
428
|
}
|
|
401
429
|
return {
|
|
402
430
|
valid: missing.length === 0 && changed.length === 0 && unexpected.length === 0,
|
|
@@ -564,13 +592,7 @@ export async function verifyTarGzipArchive(path) {
|
|
|
564
592
|
const recognizedPayload = Boolean(name)
|
|
565
593
|
&& name !== "training_args.bin"
|
|
566
594
|
&& (name.endsWith(".safetensors")
|
|
567
|
-
|| name.endsWith(".bin")
|
|
568
|
-
|| name.endsWith(".pt")
|
|
569
|
-
|| name.endsWith(".pth")
|
|
570
|
-
|| name.endsWith(".gguf")
|
|
571
|
-
|| name.endsWith(".onnx")
|
|
572
|
-
|| name.endsWith(".ckpt")
|
|
573
|
-
|| name.endsWith(".npz"));
|
|
595
|
+
|| name.endsWith(".bin"));
|
|
574
596
|
const adapterWeight = name === "adapter_model.safetensors" || name === "adapter_model.bin";
|
|
575
597
|
const fullModelWeight = Boolean(name)
|
|
576
598
|
&& ((name.endsWith(".safetensors") && !name.startsWith("adapter_"))
|