@tuned-tensor/local 0.2.8 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +84 -204
- package/dist/artifacts.d.ts +3 -4
- package/dist/artifacts.js +55 -33
- package/dist/artifacts.js.map +1 -1
- package/dist/compare.d.ts +1 -8
- package/dist/compare.js +1 -23
- package/dist/compare.js.map +1 -1
- package/dist/contracts.d.ts +59 -366
- package/dist/contracts.js +73 -224
- package/dist/contracts.js.map +1 -1
- package/dist/dataset.d.ts +2 -4
- package/dist/dataset.js +31 -125
- package/dist/dataset.js.map +1 -1
- package/dist/doctor.d.ts +2 -3
- package/dist/doctor.js +23 -161
- package/dist/doctor.js.map +1 -1
- package/dist/evaluation.d.ts +11 -71
- package/dist/evaluation.js +212 -572
- package/dist/evaluation.js.map +1 -1
- package/dist/huggingface-cache.d.ts +8 -7
- package/dist/huggingface-cache.js +16 -8
- package/dist/huggingface-cache.js.map +1 -1
- package/dist/index.d.ts +0 -10
- package/dist/index.js +205 -628
- package/dist/index.js.map +1 -1
- package/dist/local-project.d.ts +1 -11
- package/dist/local-project.js +33 -61
- package/dist/local-project.js.map +1 -1
- package/dist/model-registry.d.ts +3 -16
- package/dist/model-registry.js +76 -292
- package/dist/model-registry.js.map +1 -1
- package/dist/model-server.d.ts +1 -2
- package/dist/model-server.js +9 -18
- package/dist/model-server.js.map +1 -1
- package/dist/orchestrator.d.ts +11 -25
- package/dist/orchestrator.js +246 -566
- package/dist/orchestrator.js.map +1 -1
- package/dist/prefetch.d.ts +1 -5
- package/dist/prefetch.js +14 -19
- package/dist/prefetch.js.map +1 -1
- package/dist/process-runner.d.ts +10 -29
- package/dist/process-runner.js +63 -169
- package/dist/process-runner.js.map +1 -1
- package/dist/process-training.d.ts +0 -1
- package/dist/process-training.js +10 -87
- package/dist/process-training.js.map +1 -1
- package/dist/store.d.ts +1 -3
- package/dist/store.js +92 -290
- package/dist/store.js.map +1 -1
- package/docs/architecture.md +87 -152
- package/docs/spark.md +66 -97
- package/examples/dry-runner.json +14 -0
- package/examples/local-runner.json +8 -6
- package/examples/smoke-spec.json +41 -0
- package/package.json +6 -6
- package/training/local-runner/pyproject.toml +0 -5
- package/training/local-runner/src/evaluate.py +89 -234
- package/training/local-runner/src/model_contract.py +47 -0
- package/training/local-runner/src/prefetch.py +18 -4
- package/training/local-runner/src/serve.py +54 -115
- package/training/local-runner/src/sft_data.py +97 -0
- package/training/local-runner/src/train.py +146 -346
- package/training/local-runner/uv.lock +0 -1197
- package/dist/labeling-sanitize.d.ts +0 -31
- package/dist/labeling-sanitize.js +0 -158
- package/dist/labeling-sanitize.js.map +0 -1
- package/dist/labeling.d.ts +0 -155
- package/dist/labeling.js +0 -496
- package/dist/labeling.js.map +0 -1
- package/dist/openrouter.d.ts +0 -29
- package/dist/openrouter.js +0 -66
- package/dist/openrouter.js.map +0 -1
- package/dist/server.d.ts +0 -9
- package/dist/server.js +0 -211
- package/dist/server.js.map +0 -1
- package/docs/local-workflow-remediation-2026-07-13.md +0 -130
- package/docs/local-workflow-ux-review-2026-07-13.md +0 -458
- package/examples/dpo-preferences.jsonl +0 -2
- package/examples/dpo-run-request.json +0 -34
- package/examples/smoke-run-request.json +0 -34
- package/training/local-runner/src/train_dpo.py +0 -286
package/dist/index.js
CHANGED
|
@@ -1,38 +1,30 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { readFileSync, realpathSync } from "node:fs";
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
|
-
import {
|
|
5
|
-
import { spawn } from "node:child_process";
|
|
4
|
+
import { readFile, stat } from "node:fs/promises";
|
|
6
5
|
import { basename, dirname, join, resolve } from "node:path";
|
|
7
6
|
import { cwd } from "node:process";
|
|
8
7
|
import { fileURLToPath } from "node:url";
|
|
9
8
|
import { compareRuns } from "./compare.js";
|
|
10
|
-
import { assertArtifactManifest
|
|
11
|
-
import { fineTuneRunRequestSchema, localBehaviorSpecFileSchema, localRunnerConfigSchema
|
|
9
|
+
import { assertArtifactManifest } from "./artifacts.js";
|
|
10
|
+
import { fineTuneRunRequestSchema, localBehaviorSpecFileSchema, localRunnerConfigSchema } from "./contracts.js";
|
|
12
11
|
import { buildSystemMessage } from "./dataset.js";
|
|
13
|
-
import {
|
|
14
|
-
import { runLocalLabelingJob } from "./labeling.js";
|
|
15
|
-
import { loadLocalRunnerConfig, fingerprintLocalBaseModel, runLocalFineTuneStage, } from "./orchestrator.js";
|
|
12
|
+
import { loadLocalRunnerConfig, fingerprintLocalBaseModel, runLocalFineTune, validateLocalFineTuneInput, } from "./orchestrator.js";
|
|
16
13
|
import { runDoctor } from "./doctor.js";
|
|
17
|
-
import { assertUsableModelArtifact
|
|
14
|
+
import { assertUsableModelArtifact } from "./model-registry.js";
|
|
18
15
|
import { buildLocalModelServerLaunch, serveLocalModel } from "./model-server.js";
|
|
19
16
|
import { prefetchBaseModel } from "./prefetch.js";
|
|
20
|
-
import { createLocalStore
|
|
21
|
-
import {
|
|
22
|
-
import { DEFAULT_LOCAL_SPEC_PATH, assertLocalRunInputReady, initLocalRunnerConfigFile, initLocalSpecFile, loadLocalRunInput, runRequestFromLocalSpec, } from "./local-project.js";
|
|
17
|
+
import { createLocalStore } from "./store.js";
|
|
18
|
+
import { DEFAULT_LOCAL_SPEC_PATH, assertLocalRunInputReady, initLocalRunnerConfigFile, initLocalSpecFile, loadLocalRunInput, } from "./local-project.js";
|
|
23
19
|
import { sanitizeLogLine } from "./run-reporter.js";
|
|
24
20
|
export * from "./compare.js";
|
|
25
21
|
export * from "./contracts.js";
|
|
26
22
|
export * from "./dataset.js";
|
|
27
|
-
export * from "./labeling.js";
|
|
28
|
-
export * from "./labeling-sanitize.js";
|
|
29
23
|
export * from "./model-server.js";
|
|
30
24
|
export * from "./orchestrator.js";
|
|
31
25
|
export * from "./local-project.js";
|
|
32
|
-
export * from "./openrouter.js";
|
|
33
26
|
export * from "./prefetch.js";
|
|
34
27
|
export * from "./run-reporter.js";
|
|
35
|
-
export * from "./server.js";
|
|
36
28
|
export * from "./store.js";
|
|
37
29
|
function packageVersion() {
|
|
38
30
|
try {
|
|
@@ -48,54 +40,10 @@ export function getLocalRunnerInfo() {
|
|
|
48
40
|
return {
|
|
49
41
|
name: "tuned-tensor-local",
|
|
50
42
|
status: "local-runner-preview",
|
|
51
|
-
description: "Local fine-tuning
|
|
43
|
+
description: "Local CUDA LoRA fine-tuning with held-out base-versus-tuned evaluation.",
|
|
52
44
|
version: TT_LOCAL_VERSION,
|
|
53
45
|
};
|
|
54
46
|
}
|
|
55
|
-
/**
|
|
56
|
-
* Parses simple KEY=VALUE lines from .env content. Supports comments, blank
|
|
57
|
-
* lines, an optional `export ` prefix, and single/double quoted values. Does
|
|
58
|
-
* not support multiline values or variable expansion.
|
|
59
|
-
*/
|
|
60
|
-
export function parseDotEnv(content) {
|
|
61
|
-
const values = {};
|
|
62
|
-
for (const rawLine of content.split(/\r?\n/)) {
|
|
63
|
-
const line = rawLine.trim();
|
|
64
|
-
if (!line || line.startsWith("#"))
|
|
65
|
-
continue;
|
|
66
|
-
const match = line.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
|
|
67
|
-
if (!match)
|
|
68
|
-
continue;
|
|
69
|
-
let value = match[2].trim();
|
|
70
|
-
if ((value.startsWith("\"") && value.endsWith("\"")) || (value.startsWith("'") && value.endsWith("'"))) {
|
|
71
|
-
value = value.slice(1, -1);
|
|
72
|
-
}
|
|
73
|
-
values[match[1]] = value;
|
|
74
|
-
}
|
|
75
|
-
return values;
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Loads .env from the working directory into process.env without overriding
|
|
79
|
-
* variables that are already set, so `OPENROUTER_API_KEY` and friends work
|
|
80
|
-
* out of the box in project directories. Returns the names that were loaded.
|
|
81
|
-
*/
|
|
82
|
-
async function loadDotEnvFromCwd() {
|
|
83
|
-
let content;
|
|
84
|
-
try {
|
|
85
|
-
content = await readFile(resolve(cwd(), ".env"), "utf8");
|
|
86
|
-
}
|
|
87
|
-
catch {
|
|
88
|
-
return [];
|
|
89
|
-
}
|
|
90
|
-
const loaded = [];
|
|
91
|
-
for (const [key, value] of Object.entries(parseDotEnv(content))) {
|
|
92
|
-
if (process.env[key] === undefined) {
|
|
93
|
-
process.env[key] = value;
|
|
94
|
-
loaded.push(key);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
return loaded;
|
|
98
|
-
}
|
|
99
47
|
function readOption(argv, name) {
|
|
100
48
|
const index = argv.indexOf(name);
|
|
101
49
|
if (index !== -1)
|
|
@@ -112,18 +60,12 @@ function printHelp() {
|
|
|
112
60
|
Commands:
|
|
113
61
|
info Show package and runner information
|
|
114
62
|
init [--name "My Local Model"] [--model Qwen/Qwen3.5-2B] [--output tunedtensor.json] [--profile spark] [--force]
|
|
115
|
-
doctor [tunedtensor.json
|
|
116
|
-
validate [tunedtensor.json
|
|
117
|
-
run [tunedtensor.json
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
label <input.jsonl|input.csv> [--input-column col] [--spec tunedtensor.json] [--system-prompt "..."]
|
|
121
|
-
[--model teacher-model] [--output labeled.jsonl] [--config local-runner.json] [--dry-run]
|
|
122
|
-
serve [--config local-runner.json] [--host 127.0.0.1] [--port 8787]
|
|
123
|
-
runs list|get|events|watch|report|compare|cancel|reconcile [args] [--config local-runner.json]
|
|
63
|
+
doctor [tunedtensor.json] [--config local-runner.json]
|
|
64
|
+
validate [tunedtensor.json] [--config local-runner.json]
|
|
65
|
+
run [tunedtensor.json] [--config local-runner.json] [--dry-run] [--verbose] [--quiet]
|
|
66
|
+
serve <model-id> [--config local-runner.json] [--host 127.0.0.1] [--port 8000]
|
|
67
|
+
runs list|get|events|report|compare [args] [--config local-runner.json]
|
|
124
68
|
models list|get|verify|prefetch|verify-base|serve [args] [--config local-runner.json]
|
|
125
|
-
specs list|get|import [args] [--config local-runner.json]
|
|
126
|
-
store rebuild-index [--config local-runner.json]
|
|
127
69
|
|
|
128
70
|
Global options:
|
|
129
71
|
-h, --help Show help
|
|
@@ -134,10 +76,23 @@ The run command writes local artifacts under config.artifactRoot, defaulting to
|
|
|
134
76
|
~/.tuned-tensor-local unless config.storeRoot or TT_LOCAL_HOME is set.`);
|
|
135
77
|
}
|
|
136
78
|
const CONFIG_OPTION = { name: "--config", value: "path", description: "Local runner config JSON path" };
|
|
137
|
-
const USER_ID_OPTION = { name: "--user-id", value: "id", description: "Override the local user ID" };
|
|
138
|
-
const RUN_NUMBER_OPTION = { name: "--run-number", value: "number", description: "Override the local run number" };
|
|
139
79
|
const VERBOSE_OPTION = { name: "--verbose", description: "Stream subprocess output" };
|
|
140
80
|
const QUIET_OPTION = { name: "--quiet", description: "Suppress progress output on stderr" };
|
|
81
|
+
const MODEL_SERVE_OPTIONS = [
|
|
82
|
+
CONFIG_OPTION,
|
|
83
|
+
{ name: "--host", value: "host", description: "Bind host (localhost by default)" },
|
|
84
|
+
{ name: "--port", value: "port", description: "Bind port" },
|
|
85
|
+
{ name: "--device", value: "device", description: "cuda or cpu" },
|
|
86
|
+
{ name: "--max-tokens", value: "count", description: "Default response token limit" },
|
|
87
|
+
{ name: "--temperature", value: "number", description: "Default sampling temperature" },
|
|
88
|
+
{ name: "--top-p", value: "number", description: "Default nucleus sampling threshold" },
|
|
89
|
+
{ name: "--max-concurrent-requests", value: "count", description: "Concurrent generation limit" },
|
|
90
|
+
{ name: "--spec", value: "path", description: "Behavior spec whose instructions are enforced" },
|
|
91
|
+
{ name: "--no-spec-prompt", description: "Do not enforce the stored behavior-spec prompt" },
|
|
92
|
+
{ name: "--allow-remote", description: "Allow a non-loopback bind" },
|
|
93
|
+
{ name: "--api-key-env", value: "name", description: "Environment variable containing a bearer token" },
|
|
94
|
+
{ name: "--print-command", description: "Validate and print the launch plan without starting" },
|
|
95
|
+
];
|
|
141
96
|
const COMMAND_DEFINITIONS = {
|
|
142
97
|
info: {
|
|
143
98
|
usage: "tt-local info",
|
|
@@ -153,86 +108,53 @@ const COMMAND_DEFINITIONS = {
|
|
|
153
108
|
{ name: "--model", value: "model", description: "Base model ID" },
|
|
154
109
|
{ name: "--output", value: "path", description: "Output spec path" },
|
|
155
110
|
{ name: "--profile", value: "profile", description: "Write a durable runner config (spark)" },
|
|
156
|
-
{ name: "--config
|
|
111
|
+
{ name: "--config", value: "path", description: "Runner config path (written with --profile)" },
|
|
157
112
|
{ name: "--force", description: "Overwrite an existing output file" },
|
|
158
113
|
],
|
|
159
114
|
maxPositionals: 0,
|
|
160
115
|
},
|
|
161
116
|
doctor: {
|
|
162
|
-
usage: "tt-local doctor [tunedtensor.json
|
|
117
|
+
usage: "tt-local doctor [tunedtensor.json] [--config path]",
|
|
163
118
|
description: "Check the host and optional run input before starting work.",
|
|
164
119
|
options: [CONFIG_OPTION],
|
|
165
120
|
maxPositionals: 1,
|
|
166
121
|
},
|
|
167
122
|
validate: {
|
|
168
|
-
usage: "tt-local validate [tunedtensor.json
|
|
169
|
-
description: "Validate a behavior spec
|
|
170
|
-
options: [CONFIG_OPTION
|
|
123
|
+
usage: "tt-local validate [tunedtensor.json] [options]",
|
|
124
|
+
description: "Validate a local behavior spec without executing it.",
|
|
125
|
+
options: [CONFIG_OPTION],
|
|
171
126
|
maxPositionals: 1,
|
|
172
127
|
},
|
|
173
128
|
run: {
|
|
174
|
-
usage: "tt-local run [tunedtensor.json
|
|
175
|
-
description: "Run the
|
|
129
|
+
usage: "tt-local run [tunedtensor.json] [options]",
|
|
130
|
+
description: "Run the baseline, fine-tuning, tuned evaluation, and report workflow.",
|
|
176
131
|
options: [
|
|
177
132
|
CONFIG_OPTION,
|
|
178
|
-
USER_ID_OPTION,
|
|
179
|
-
RUN_NUMBER_OPTION,
|
|
180
|
-
{ name: "--run-id", value: "uuid", description: "Resume or override a run ID" },
|
|
181
|
-
{ name: "--stage", value: "stage", description: "Run through a specific workflow stage" },
|
|
182
|
-
{ name: "--model-artifact", value: "uri", description: "Use an existing model artifact" },
|
|
183
|
-
{ name: "--parent-model", value: "model-id", description: "Continue training a stored local model" },
|
|
184
|
-
{ name: "--parent-model-artifact", value: "uri", description: "Continue training a model artifact" },
|
|
185
|
-
{ name: "--force", description: "Recompute the selected stage" },
|
|
186
133
|
{ name: "--dry-run", description: "Write representative artifacts without GPU work" },
|
|
187
|
-
{ name: "--detach", description: "Run in the background and return the run ID immediately" },
|
|
188
134
|
VERBOSE_OPTION,
|
|
189
135
|
QUIET_OPTION,
|
|
190
136
|
],
|
|
191
137
|
maxPositionals: 1,
|
|
192
138
|
},
|
|
193
|
-
|
|
194
|
-
usage: "tt-local
|
|
195
|
-
description: "
|
|
196
|
-
options:
|
|
197
|
-
CONFIG_OPTION,
|
|
198
|
-
{ name: "--input-column", value: "column", description: "CSV input column" },
|
|
199
|
-
{ name: "--spec", value: "path", description: "Behavior spec path" },
|
|
200
|
-
{ name: "--system-prompt", value: "text", description: "Explicit teacher system prompt" },
|
|
201
|
-
{ name: "--model", value: "model", description: "Teacher model ID" },
|
|
202
|
-
{ name: "--output", value: "path", description: "Output JSONL path" },
|
|
203
|
-
{ name: "--format", value: "jsonl|csv", description: "Force the source format" },
|
|
204
|
-
{ name: "--dry-run", description: "Parse and sanitize without teacher calls" },
|
|
205
|
-
VERBOSE_OPTION,
|
|
206
|
-
QUIET_OPTION,
|
|
207
|
-
],
|
|
139
|
+
serve: {
|
|
140
|
+
usage: "tt-local serve <model-id> [options]",
|
|
141
|
+
description: "Serve a verified model through an OpenAI-compatible local API.",
|
|
142
|
+
options: MODEL_SERVE_OPTIONS,
|
|
208
143
|
minPositionals: 1,
|
|
209
144
|
maxPositionals: 1,
|
|
210
|
-
missingPositionalsMessage: "
|
|
211
|
-
},
|
|
212
|
-
serve: {
|
|
213
|
-
usage: "tt-local serve [--config path] [--host host] [--port port]",
|
|
214
|
-
description: "Serve the local runs dashboard.",
|
|
215
|
-
options: [
|
|
216
|
-
CONFIG_OPTION,
|
|
217
|
-
{ name: "--host", value: "host", description: "Dashboard bind host" },
|
|
218
|
-
{ name: "--port", value: "port", description: "Dashboard bind port" },
|
|
219
|
-
],
|
|
220
|
-
maxPositionals: 0,
|
|
145
|
+
missingPositionalsMessage: "serve requires <model-id>",
|
|
221
146
|
},
|
|
222
147
|
};
|
|
223
148
|
const COMMAND_GROUPS = {
|
|
224
149
|
runs: {
|
|
225
|
-
description: "Inspect
|
|
150
|
+
description: "Inspect locally stored runs.",
|
|
226
151
|
defaultSubcommand: "list",
|
|
227
152
|
subcommands: {
|
|
228
153
|
list: { usage: "tt-local runs list [--config path]", description: "List local runs.", options: [CONFIG_OPTION], maxPositionals: 0 },
|
|
229
154
|
get: { usage: "tt-local runs get <run-id> [--config path]", description: "Get a local run.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs get requires <run-id>" },
|
|
230
155
|
events: { usage: "tt-local runs events <run-id> [--config path]", description: "List run events.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs events requires <run-id>" },
|
|
231
|
-
|
|
232
|
-
report: { usage: "tt-local runs report <run-id> [--config path]", description: "Get a run comparison report.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs report requires <run-id>" },
|
|
156
|
+
report: { usage: "tt-local runs report <run-id> [--config path]", description: "Show the baseline-vs-tuned report, including deltas and regressions.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs report requires <run-id>" },
|
|
233
157
|
compare: { usage: "tt-local runs compare <run-id-a> <run-id-b> [--config path]", description: "Compare two run reports.", options: [CONFIG_OPTION], minPositionals: 2, maxPositionals: 2, missingPositionalsMessage: "runs compare requires <run-id-a> <run-id-b>" },
|
|
234
|
-
cancel: { usage: "tt-local runs cancel <run-id> [--config path]", description: "Request cancellation of a run.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "runs cancel requires <run-id>" },
|
|
235
|
-
reconcile: { usage: "tt-local runs reconcile [--config path]", description: "Rebuild local store indexes.", options: [CONFIG_OPTION], maxPositionals: 0 },
|
|
236
158
|
},
|
|
237
159
|
},
|
|
238
160
|
models: {
|
|
@@ -250,63 +172,27 @@ const COMMAND_GROUPS = {
|
|
|
250
172
|
missingPositionalsMessage: "models verify requires <model-id-or-artifact-path>",
|
|
251
173
|
},
|
|
252
174
|
prefetch: {
|
|
253
|
-
usage: "tt-local models prefetch [tunedtensor.json
|
|
175
|
+
usage: "tt-local models prefetch [tunedtensor.json] [options]",
|
|
254
176
|
description: "Download the configured base model before a run.",
|
|
255
|
-
options: [CONFIG_OPTION,
|
|
177
|
+
options: [CONFIG_OPTION, VERBOSE_OPTION, QUIET_OPTION],
|
|
256
178
|
maxPositionals: 1,
|
|
257
179
|
},
|
|
258
180
|
"verify-base": {
|
|
259
|
-
usage: "tt-local models verify-base [tunedtensor.json
|
|
181
|
+
usage: "tt-local models verify-base [tunedtensor.json] [options]",
|
|
260
182
|
description: "Verify that the configured base-model snapshot is complete and locally available.",
|
|
261
|
-
options: [CONFIG_OPTION,
|
|
183
|
+
options: [CONFIG_OPTION, VERBOSE_OPTION, QUIET_OPTION],
|
|
262
184
|
maxPositionals: 1,
|
|
263
185
|
},
|
|
264
186
|
serve: {
|
|
265
187
|
usage: "tt-local models serve <model-id> [options]",
|
|
266
|
-
description: "
|
|
267
|
-
options:
|
|
268
|
-
CONFIG_OPTION,
|
|
269
|
-
{ name: "--host", value: "host", description: "Bind host (localhost by default)" },
|
|
270
|
-
{ name: "--port", value: "port", description: "Bind port" },
|
|
271
|
-
{ name: "--device", value: "device", description: "auto, cpu, cuda, or mps" },
|
|
272
|
-
{ name: "--max-tokens", value: "count", description: "Default response token limit" },
|
|
273
|
-
{ name: "--temperature", value: "number", description: "Default sampling temperature" },
|
|
274
|
-
{ name: "--top-p", value: "number", description: "Default nucleus sampling threshold" },
|
|
275
|
-
{ name: "--max-concurrent-requests", value: "count", description: "Concurrent generation limit" },
|
|
276
|
-
{ name: "--spec", value: "path", description: "Behavior spec whose instructions are enforced" },
|
|
277
|
-
{ name: "--no-spec-prompt", description: "Do not enforce the stored behavior-spec prompt" },
|
|
278
|
-
{ name: "--allow-remote", description: "Allow a non-loopback bind" },
|
|
279
|
-
{ name: "--api-key-env", value: "name", description: "Environment variable containing a bearer token" },
|
|
280
|
-
{ name: "--print-command", description: "Validate and print the launch plan without starting" },
|
|
281
|
-
],
|
|
188
|
+
description: "Alias for `tt-local serve`.",
|
|
189
|
+
options: MODEL_SERVE_OPTIONS,
|
|
282
190
|
minPositionals: 1,
|
|
283
191
|
maxPositionals: 1,
|
|
284
192
|
missingPositionalsMessage: "models serve requires <model-id>",
|
|
285
193
|
},
|
|
286
194
|
},
|
|
287
195
|
},
|
|
288
|
-
specs: {
|
|
289
|
-
description: "Inspect and import local behavior specs.",
|
|
290
|
-
defaultSubcommand: "list",
|
|
291
|
-
subcommands: {
|
|
292
|
-
list: { usage: "tt-local specs list [--config path]", description: "List local behavior specs.", options: [CONFIG_OPTION], maxPositionals: 0 },
|
|
293
|
-
get: { usage: "tt-local specs get <spec-id> [--config path]", description: "Get a local behavior spec.", options: [CONFIG_OPTION], minPositionals: 1, maxPositionals: 1, missingPositionalsMessage: "specs get requires <spec-id>" },
|
|
294
|
-
import: {
|
|
295
|
-
usage: "tt-local specs import <spec-or-request.json> [options]",
|
|
296
|
-
description: "Import a behavior spec into the local store.",
|
|
297
|
-
options: [CONFIG_OPTION, USER_ID_OPTION, RUN_NUMBER_OPTION, { name: "--id", value: "id", description: "ID for a raw spec snapshot" }],
|
|
298
|
-
minPositionals: 1,
|
|
299
|
-
maxPositionals: 1,
|
|
300
|
-
missingPositionalsMessage: "specs import requires <spec-or-request.json>",
|
|
301
|
-
},
|
|
302
|
-
},
|
|
303
|
-
},
|
|
304
|
-
store: {
|
|
305
|
-
description: "Maintain the local file-backed store.",
|
|
306
|
-
subcommands: {
|
|
307
|
-
"rebuild-index": { usage: "tt-local store rebuild-index [--config path]", description: "Rebuild store indexes from durable records.", options: [CONFIG_OPTION], maxPositionals: 0 },
|
|
308
|
-
},
|
|
309
|
-
},
|
|
310
196
|
};
|
|
311
197
|
function hasHelpFlag(argv) {
|
|
312
198
|
return argv.includes("--help") || argv.includes("-h");
|
|
@@ -439,22 +325,32 @@ function readNumberOption(argv, name) {
|
|
|
439
325
|
const value = readOption(argv, name);
|
|
440
326
|
return value ? Number(value) : undefined;
|
|
441
327
|
}
|
|
442
|
-
function
|
|
443
|
-
const
|
|
444
|
-
if (
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|| value === "report"
|
|
450
|
-
|| value === "all") {
|
|
451
|
-
return value;
|
|
452
|
-
}
|
|
453
|
-
throw new Error(`--stage must be one of prepare, baseline, train, candidate, score, report, all; got: ${value}`);
|
|
328
|
+
async function selectedConfigPath(argv, adjacentTo) {
|
|
329
|
+
const explicitPath = readOption(argv, "--config");
|
|
330
|
+
if (explicitPath)
|
|
331
|
+
return resolve(explicitPath);
|
|
332
|
+
const candidate = join(adjacentTo ? dirname(resolve(adjacentTo)) : cwd(), "local-runner.json");
|
|
333
|
+
const metadata = await stat(candidate).catch(() => null);
|
|
334
|
+
return metadata?.isFile() ? candidate : undefined;
|
|
454
335
|
}
|
|
455
|
-
async function
|
|
456
|
-
const
|
|
457
|
-
return
|
|
336
|
+
async function configSelectionFromArgv(argv, adjacentTo) {
|
|
337
|
+
const path = await selectedConfigPath(argv, adjacentTo);
|
|
338
|
+
return {
|
|
339
|
+
config: await loadLocalRunnerConfig(path),
|
|
340
|
+
...(path ? { path } : {}),
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
async function configFromArgv(argv, adjacentTo) {
|
|
344
|
+
return (await configSelectionFromArgv(argv, adjacentTo)).config;
|
|
345
|
+
}
|
|
346
|
+
async function loadCliBehaviorSpec(inputPath, runId) {
|
|
347
|
+
const input = await loadLocalRunInput(inputPath, {
|
|
348
|
+
...(runId ? { runId } : {}),
|
|
349
|
+
});
|
|
350
|
+
if (input.kind !== "spec") {
|
|
351
|
+
throw new Error(`TT Local CLI expects a tunedtensor.json behavior spec, not a full run request: ${input.path}`);
|
|
352
|
+
}
|
|
353
|
+
return input;
|
|
458
354
|
}
|
|
459
355
|
function printJson(value) {
|
|
460
356
|
console.log(JSON.stringify(value, null, 2));
|
|
@@ -503,162 +399,7 @@ function createConsoleReporter(options) {
|
|
|
503
399
|
},
|
|
504
400
|
};
|
|
505
401
|
}
|
|
506
|
-
function
|
|
507
|
-
if (request.training_method !== "dpo" || config.training.backend === "command")
|
|
508
|
-
return;
|
|
509
|
-
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
510
|
-
if (model.loader === "image_text_to_text") {
|
|
511
|
-
throw new Error("Bundled DPO training supports text causal-LM models only in v1. Use a causal_lm base model or training.backend=command for a custom DPO trainer.");
|
|
512
|
-
}
|
|
513
|
-
}
|
|
514
|
-
async function parentModelSelectionFromArgv(argv, config) {
|
|
515
|
-
const parentModel = readOption(argv, "--parent-model");
|
|
516
|
-
const parentModelArtifact = readOption(argv, "--parent-model-artifact");
|
|
517
|
-
if (parentModel && parentModelArtifact) {
|
|
518
|
-
throw new Error("Use only one of --parent-model or --parent-model-artifact.");
|
|
519
|
-
}
|
|
520
|
-
if (parentModelArtifact)
|
|
521
|
-
return { artifactUri: parentModelArtifact };
|
|
522
|
-
if (!parentModel)
|
|
523
|
-
return undefined;
|
|
524
|
-
const store = createLocalStore(config.storeRoot);
|
|
525
|
-
const record = await store.getModel(parentModel);
|
|
526
|
-
const verified = await verifyStoredModel(record, { requireServable: true });
|
|
527
|
-
const contract = verified.contract;
|
|
528
|
-
const baseModelRevision = typeof contract.base_model_revision === "string"
|
|
529
|
-
? contract.base_model_revision
|
|
530
|
-
: undefined;
|
|
531
|
-
const baseModelArtifactUri = typeof contract.base_model_artifact_uri === "string"
|
|
532
|
-
? contract.base_model_artifact_uri
|
|
533
|
-
: undefined;
|
|
534
|
-
const baseModelFingerprint = typeof contract.base_model_fingerprint === "string"
|
|
535
|
-
? contract.base_model_fingerprint
|
|
536
|
-
: undefined;
|
|
537
|
-
if (baseModelArtifactUri && baseModelFingerprint) {
|
|
538
|
-
if (!config.paths.baseModel) {
|
|
539
|
-
throw new Error(`Parent model ${record.id} was trained from a local base snapshot. Configure paths.baseModel to the same `
|
|
540
|
-
+ "snapshot before continuing it.");
|
|
541
|
-
}
|
|
542
|
-
const configuredFingerprint = await fingerprintLocalBaseModel(config.paths.baseModel);
|
|
543
|
-
if (configuredFingerprint !== baseModelFingerprint) {
|
|
544
|
-
throw new Error(`Configured paths.baseModel content does not match parent model ${record.id}.`);
|
|
545
|
-
}
|
|
546
|
-
}
|
|
547
|
-
else if (!baseModelRevision) {
|
|
548
|
-
throw new Error(`Parent model ${record.id} does not record an immutable base-model revision or local snapshot fingerprint. `
|
|
549
|
-
+ "Use --parent-model-artifact only if you intentionally accept responsibility for that unpinned artifact.");
|
|
550
|
-
}
|
|
551
|
-
if (baseModelRevision && config.paths.baseModel) {
|
|
552
|
-
const snapshotRevision = resolve(config.paths.baseModel).match(/[\\/]snapshots[\\/]([^\\/]+)(?:[\\/]|$)/)?.[1];
|
|
553
|
-
if (!baseModelFingerprint && snapshotRevision !== baseModelRevision) {
|
|
554
|
-
throw new Error(`Parent model ${record.id} requires base revision ${baseModelRevision}; configured paths.baseModel does not `
|
|
555
|
-
+ "identify that snapshot revision.");
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
return {
|
|
559
|
-
artifactUri: record.artifact_uri,
|
|
560
|
-
modelId: record.id,
|
|
561
|
-
baseModel: record.base_model,
|
|
562
|
-
baseModelRevision,
|
|
563
|
-
baseModelArtifactUri,
|
|
564
|
-
baseModelFingerprint,
|
|
565
|
-
};
|
|
566
|
-
}
|
|
567
|
-
function withParentModelArtifact(request, parentModel) {
|
|
568
|
-
if (!parentModel)
|
|
569
|
-
return request;
|
|
570
|
-
if (parentModel.baseModel && parentModel.baseModel !== request.spec_snapshot.base_model) {
|
|
571
|
-
throw new Error(`Parent model ${parentModel.modelId ?? ""} uses base model ${parentModel.baseModel}, `
|
|
572
|
-
+ `but this run uses ${request.spec_snapshot.base_model}. Continue from a model with the same base model.`);
|
|
573
|
-
}
|
|
574
|
-
const requestedRevision = request.hyperparameters.base_model_revision;
|
|
575
|
-
if (parentModel.baseModelRevision && requestedRevision && parentModel.baseModelRevision !== requestedRevision) {
|
|
576
|
-
throw new Error(`Parent model ${parentModel.modelId ?? ""} uses base revision ${parentModel.baseModelRevision}, `
|
|
577
|
-
+ `but this run requests ${requestedRevision}.`);
|
|
578
|
-
}
|
|
579
|
-
return fineTuneRunRequestSchema.parse({
|
|
580
|
-
...request,
|
|
581
|
-
hyperparameters: {
|
|
582
|
-
...request.hyperparameters,
|
|
583
|
-
...(parentModel.baseModelRevision ? { base_model_revision: parentModel.baseModelRevision } : {}),
|
|
584
|
-
parent_model_artifact: parentModel.artifactUri,
|
|
585
|
-
},
|
|
586
|
-
});
|
|
587
|
-
}
|
|
588
|
-
async function sleep(ms) {
|
|
589
|
-
await new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
|
590
|
-
}
|
|
591
|
-
function detachedRunArgv(argv, runId) {
|
|
592
|
-
const tokens = argv.slice(3);
|
|
593
|
-
const output = [];
|
|
594
|
-
for (let index = 0; index < tokens.length; index += 1) {
|
|
595
|
-
const token = tokens[index];
|
|
596
|
-
if (token === "--detach")
|
|
597
|
-
continue;
|
|
598
|
-
if (token === "--run-id") {
|
|
599
|
-
index += 1;
|
|
600
|
-
continue;
|
|
601
|
-
}
|
|
602
|
-
if (token.startsWith("--run-id="))
|
|
603
|
-
continue;
|
|
604
|
-
output.push(token);
|
|
605
|
-
}
|
|
606
|
-
return ["run", ...output, "--run-id", runId];
|
|
607
|
-
}
|
|
608
|
-
async function launchDetachedRun(args) {
|
|
609
|
-
if (!process.argv[1])
|
|
610
|
-
throw new Error("Cannot determine the tt-local CLI entrypoint for --detach.");
|
|
611
|
-
const prefix = args.request.artifacts?.prefix ?? defaultArtifactPrefix({
|
|
612
|
-
userId: args.request.user_id,
|
|
613
|
-
behaviorSpecId: args.request.behavior_spec_id,
|
|
614
|
-
runId: args.request.run_id,
|
|
615
|
-
});
|
|
616
|
-
const artifacts = resolveRunArtifacts({ artifactRoot: args.config.artifactRoot, prefix });
|
|
617
|
-
await claimRunArtifactDirectory({
|
|
618
|
-
artifacts,
|
|
619
|
-
runId: args.request.run_id,
|
|
620
|
-
userId: args.request.user_id,
|
|
621
|
-
behaviorSpecId: args.request.behavior_spec_id,
|
|
622
|
-
});
|
|
623
|
-
const store = createLocalStore(args.config.storeRoot);
|
|
624
|
-
const existing = await store.getRun(args.request.run_id).catch(() => null);
|
|
625
|
-
if (existing) {
|
|
626
|
-
await store.syncRunRequest(args.request);
|
|
627
|
-
}
|
|
628
|
-
else {
|
|
629
|
-
await store.startRun({ request: args.request, artifactDir: artifacts.runDir });
|
|
630
|
-
}
|
|
631
|
-
const logPath = resolve(artifacts.runDir, "detached.log");
|
|
632
|
-
await mkdir(dirname(logPath), { recursive: true });
|
|
633
|
-
const log = await open(logPath, "a");
|
|
634
|
-
try {
|
|
635
|
-
const child = spawn(process.execPath, [
|
|
636
|
-
...process.execArgv,
|
|
637
|
-
process.argv[1],
|
|
638
|
-
...detachedRunArgv(args.argv, args.request.run_id),
|
|
639
|
-
], {
|
|
640
|
-
detached: true,
|
|
641
|
-
stdio: ["ignore", log.fd, log.fd],
|
|
642
|
-
env: { ...process.env, TT_LOCAL_DETACHED: "1" },
|
|
643
|
-
});
|
|
644
|
-
await new Promise((resolveSpawn, reject) => {
|
|
645
|
-
child.once("spawn", resolveSpawn);
|
|
646
|
-
child.once("error", reject);
|
|
647
|
-
});
|
|
648
|
-
if (!child.pid)
|
|
649
|
-
throw new Error("Detached tt-local process did not report a PID.");
|
|
650
|
-
child.unref();
|
|
651
|
-
return { pid: child.pid, logPath, artifactDir: artifacts.runDir };
|
|
652
|
-
}
|
|
653
|
-
catch (error) {
|
|
654
|
-
await store.failRun(args.request.run_id, `Failed to launch detached workflow: ${error instanceof Error ? error.message : String(error)}`).catch(() => undefined);
|
|
655
|
-
throw error;
|
|
656
|
-
}
|
|
657
|
-
finally {
|
|
658
|
-
await log.close();
|
|
659
|
-
}
|
|
660
|
-
}
|
|
661
|
-
async function verifyStoredModel(model, options = {}) {
|
|
402
|
+
async function verifyStoredModel(model) {
|
|
662
403
|
const manifestPath = join(model.artifact_dir, "artifact-manifest.json");
|
|
663
404
|
const integrity = await assertArtifactManifest(manifestPath, {
|
|
664
405
|
requiredPaths: ["stage-metadata.json", "training-report.json"],
|
|
@@ -669,11 +410,7 @@ async function verifyStoredModel(model, options = {}) {
|
|
|
669
410
|
if (!manifest.model) {
|
|
670
411
|
throw new Error(`Artifact manifest does not contain a model contract: ${manifestPath}`);
|
|
671
412
|
}
|
|
672
|
-
const
|
|
673
|
-
&& manifest.model.servable === false;
|
|
674
|
-
const artifact = await assertUsableModelArtifact(model.artifact_uri, {
|
|
675
|
-
allowUnrecognizedPayload: customNonServable && !options.requireServable,
|
|
676
|
-
});
|
|
413
|
+
const artifact = await assertUsableModelArtifact(model.artifact_uri);
|
|
677
414
|
if (typeof manifest.model.base_model_artifact_uri === "string"
|
|
678
415
|
&& typeof manifest.model.base_model_fingerprint === "string") {
|
|
679
416
|
const actualBaseFingerprint = await fingerprintLocalBaseModel(manifest.model.base_model_artifact_uri);
|
|
@@ -688,9 +425,6 @@ async function verifyStoredModel(model, options = {}) {
|
|
|
688
425
|
if (manifest.model.base_model !== model.base_model) {
|
|
689
426
|
throw new Error("Stored model base model does not match its artifact manifest.");
|
|
690
427
|
}
|
|
691
|
-
if (options.requireServable && manifest.model.servable !== true) {
|
|
692
|
-
throw new Error(`Model ${model.id} is valid but its artifact contract does not mark it as locally servable.`);
|
|
693
|
-
}
|
|
694
428
|
return {
|
|
695
429
|
manifest_path: manifestPath,
|
|
696
430
|
integrity,
|
|
@@ -743,11 +477,7 @@ async function verifyModelArtifactPath(input) {
|
|
|
743
477
|
const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
|
744
478
|
if (!manifest.model)
|
|
745
479
|
throw new Error(`Artifact manifest does not contain a model contract: ${manifestPath}`);
|
|
746
|
-
const
|
|
747
|
-
&& manifest.model.servable === false;
|
|
748
|
-
const artifact = await assertUsableModelArtifact(artifactUri, {
|
|
749
|
-
allowUnrecognizedPayload: customNonServable,
|
|
750
|
-
});
|
|
480
|
+
const artifact = await assertUsableModelArtifact(artifactUri);
|
|
751
481
|
if (typeof manifest.model.base_model_artifact_uri === "string"
|
|
752
482
|
&& typeof manifest.model.base_model_fingerprint === "string") {
|
|
753
483
|
const actualBaseFingerprint = await fingerprintLocalBaseModel(manifest.model.base_model_artifact_uri);
|
|
@@ -772,12 +502,10 @@ async function modelSystemPrompt(args) {
|
|
|
772
502
|
if (specPath) {
|
|
773
503
|
const input = JSON.parse(await readFile(resolve(specPath), "utf8"));
|
|
774
504
|
const local = localBehaviorSpecFileSchema.safeParse(input);
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
throw new Error(`--spec must contain a TT Local behavior spec or run request: ${resolve(specPath)}`);
|
|
780
|
-
spec = parsedSpec;
|
|
505
|
+
if (!local.success) {
|
|
506
|
+
throw new Error(`--spec must contain a tunedtensor.json behavior spec: ${resolve(specPath)}`);
|
|
507
|
+
}
|
|
508
|
+
spec = local.data;
|
|
781
509
|
}
|
|
782
510
|
else {
|
|
783
511
|
const runRequestPath = join(args.store.paths.runsDir, args.model.run_id, "request.json");
|
|
@@ -806,10 +534,54 @@ async function modelSystemPrompt(args) {
|
|
|
806
534
|
}
|
|
807
535
|
function modelServeDevice(argv) {
|
|
808
536
|
const value = readOption(argv, "--device");
|
|
809
|
-
if (value === undefined || value === "
|
|
537
|
+
if (value === undefined || value === "cpu" || value === "cuda") {
|
|
810
538
|
return value;
|
|
811
539
|
}
|
|
812
|
-
throw new Error(`--device must be
|
|
540
|
+
throw new Error(`--device must be cuda or cpu; got: ${value}`);
|
|
541
|
+
}
|
|
542
|
+
async function serveStoredModelFromCli(args) {
|
|
543
|
+
const store = createLocalStore(args.config.storeRoot);
|
|
544
|
+
const model = await store.getModel(args.modelId);
|
|
545
|
+
const verified = await verifyStoredModel(model);
|
|
546
|
+
const launch = buildLocalModelServerLaunch({
|
|
547
|
+
model,
|
|
548
|
+
config: args.config,
|
|
549
|
+
options: {
|
|
550
|
+
host: readOption(args.argv, "--host"),
|
|
551
|
+
port: readNumberOption(args.argv, "--port"),
|
|
552
|
+
device: modelServeDevice(args.argv),
|
|
553
|
+
maxTokens: readNumberOption(args.argv, "--max-tokens"),
|
|
554
|
+
temperature: readNumberOption(args.argv, "--temperature"),
|
|
555
|
+
topP: readNumberOption(args.argv, "--top-p"),
|
|
556
|
+
maxConcurrentRequests: readNumberOption(args.argv, "--max-concurrent-requests"),
|
|
557
|
+
systemPrompt: await modelSystemPrompt({ argv: args.argv, model, store }),
|
|
558
|
+
allowRemote: hasFlag(args.argv, "--allow-remote"),
|
|
559
|
+
apiKeyEnv: readOption(args.argv, "--api-key-env"),
|
|
560
|
+
baseModelRevision: (() => {
|
|
561
|
+
const contract = verified.contract;
|
|
562
|
+
return typeof contract.base_model_revision === "string" ? contract.base_model_revision : undefined;
|
|
563
|
+
})(),
|
|
564
|
+
baseModelArtifactUri: (() => {
|
|
565
|
+
const contract = verified.contract;
|
|
566
|
+
return typeof contract.base_model_artifact_uri === "string" ? contract.base_model_artifact_uri : undefined;
|
|
567
|
+
})(),
|
|
568
|
+
},
|
|
569
|
+
});
|
|
570
|
+
if (hasFlag(args.argv, "--print-command")) {
|
|
571
|
+
printJson({
|
|
572
|
+
ok: true,
|
|
573
|
+
model_id: model.id,
|
|
574
|
+
url: launch.url,
|
|
575
|
+
command: launch.displayCommand,
|
|
576
|
+
artifact_path: launch.artifactPath,
|
|
577
|
+
manifest_path: verified.manifest_path,
|
|
578
|
+
integrity: verified.integrity,
|
|
579
|
+
});
|
|
580
|
+
return;
|
|
581
|
+
}
|
|
582
|
+
process.stderr.write(`[tt-local] verified ${verified.integrity.checked} artifact file(s)\n`);
|
|
583
|
+
process.stderr.write(`[tt-local] model API: ${launch.url}\n`);
|
|
584
|
+
await serveLocalModel(launch);
|
|
813
585
|
}
|
|
814
586
|
async function main(argv) {
|
|
815
587
|
const cli = parseCli(argv);
|
|
@@ -830,10 +602,6 @@ async function main(argv) {
|
|
|
830
602
|
printCommandHelp(cli.definition);
|
|
831
603
|
return;
|
|
832
604
|
}
|
|
833
|
-
const loadedEnv = await loadDotEnvFromCwd();
|
|
834
|
-
if (loadedEnv.length > 0 && !hasFlag(argv, "--quiet")) {
|
|
835
|
-
process.stderr.write(`[tt-local] loaded ${loadedEnv.join(", ")} from .env\n`);
|
|
836
|
-
}
|
|
837
605
|
if (command === "info") {
|
|
838
606
|
const info = getLocalRunnerInfo();
|
|
839
607
|
console.log(`${info.name}: ${info.description}`);
|
|
@@ -842,13 +610,21 @@ async function main(argv) {
|
|
|
842
610
|
return;
|
|
843
611
|
}
|
|
844
612
|
if (command === "doctor") {
|
|
845
|
-
const
|
|
846
|
-
const
|
|
847
|
-
|
|
613
|
+
const inputPath = resolve(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH);
|
|
614
|
+
const configSelection = await configSelectionFromArgv(argv, inputPath);
|
|
615
|
+
const hasDefaultSpec = cli.positionals[0]
|
|
616
|
+
? true
|
|
617
|
+
: Boolean((await stat(inputPath).catch(() => null))?.isFile());
|
|
618
|
+
const request = hasDefaultSpec
|
|
619
|
+
? (await loadCliBehaviorSpec(inputPath)).request
|
|
848
620
|
: undefined;
|
|
849
|
-
const checks = await runDoctor(config, request);
|
|
621
|
+
const checks = await runDoctor(configSelection.config, request);
|
|
850
622
|
const ok = checks.every((check) => check.ok);
|
|
851
|
-
printJson({
|
|
623
|
+
printJson({
|
|
624
|
+
ok,
|
|
625
|
+
config_path: configSelection.path ?? null,
|
|
626
|
+
checks,
|
|
627
|
+
});
|
|
852
628
|
if (!ok)
|
|
853
629
|
process.exitCode = 1;
|
|
854
630
|
return;
|
|
@@ -866,15 +642,18 @@ async function main(argv) {
|
|
|
866
642
|
force: hasFlag(argv, "--force"),
|
|
867
643
|
});
|
|
868
644
|
const configPath = profile
|
|
869
|
-
? resolve(readOption(argv, "--config
|
|
870
|
-
:
|
|
871
|
-
if (profile
|
|
645
|
+
? resolve(readOption(argv, "--config") ?? resolve(dirname(outputPath), "local-runner.json"))
|
|
646
|
+
: await selectedConfigPath(argv, outputPath);
|
|
647
|
+
if (profile) {
|
|
872
648
|
await initLocalRunnerConfigFile({
|
|
873
649
|
outputPath: configPath,
|
|
874
650
|
profile,
|
|
875
651
|
force: hasFlag(argv, "--force"),
|
|
876
652
|
});
|
|
877
653
|
}
|
|
654
|
+
else if (configPath) {
|
|
655
|
+
await loadLocalRunnerConfig(configPath);
|
|
656
|
+
}
|
|
878
657
|
printJson({
|
|
879
658
|
ok: true,
|
|
880
659
|
path: outputPath,
|
|
@@ -887,174 +666,94 @@ async function main(argv) {
|
|
|
887
666
|
}
|
|
888
667
|
if (command === "validate") {
|
|
889
668
|
const inputPath = resolve(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH);
|
|
890
|
-
const input = await
|
|
891
|
-
|
|
892
|
-
|
|
669
|
+
const input = await loadCliBehaviorSpec(inputPath);
|
|
670
|
+
assertLocalRunInputReady(input.request);
|
|
671
|
+
const configSelection = await configSelectionFromArgv(argv, inputPath);
|
|
672
|
+
const validated = await validateLocalFineTuneInput({
|
|
673
|
+
request: input.request,
|
|
674
|
+
config: configSelection.config,
|
|
893
675
|
});
|
|
894
|
-
const request =
|
|
895
|
-
const config =
|
|
896
|
-
assertLocalRunInputReady(request);
|
|
897
|
-
assertSupportedValidateShape(request, config);
|
|
898
|
-
assertEvaluationScoringReady(config);
|
|
676
|
+
const request = validated.request;
|
|
677
|
+
const config = validated.config;
|
|
899
678
|
printJson({
|
|
900
679
|
ok: true,
|
|
901
|
-
input_kind: input.kind,
|
|
902
680
|
input_path: input.path,
|
|
903
|
-
|
|
681
|
+
config_path: configSelection.path ?? null,
|
|
904
682
|
behavior_spec_id: request.behavior_spec_id,
|
|
905
|
-
training_method: request.training_method,
|
|
906
683
|
base_model: request.spec_snapshot.base_model,
|
|
907
684
|
dataset_format: request.dataset_prebuilt?.format ?? null,
|
|
908
685
|
artifact_root: config.artifactRoot,
|
|
909
686
|
store_root: config.storeRoot,
|
|
910
687
|
dry_run: config.dryRun,
|
|
911
|
-
...(input.warnings.length > 0 ? { warnings: input.warnings } : {}),
|
|
912
688
|
});
|
|
913
689
|
return;
|
|
914
690
|
}
|
|
915
691
|
if (command === "run") {
|
|
916
692
|
const inputPath = resolve(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH);
|
|
917
|
-
const
|
|
693
|
+
const configSelection = await configSelectionFromArgv(argv, inputPath);
|
|
694
|
+
const configInput = configSelection.config;
|
|
918
695
|
const config = localRunnerConfigSchema.parse({
|
|
919
696
|
...configInput,
|
|
920
697
|
dryRun: hasFlag(argv, "--dry-run") ? true : configInput.dryRun,
|
|
921
698
|
});
|
|
922
|
-
const input = await
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
699
|
+
const input = await loadCliBehaviorSpec(inputPath);
|
|
700
|
+
assertLocalRunInputReady(input.request);
|
|
701
|
+
const validated = await validateLocalFineTuneInput({
|
|
702
|
+
request: input.request,
|
|
703
|
+
config,
|
|
704
|
+
});
|
|
705
|
+
let request = validated.request;
|
|
706
|
+
const reporter = createConsoleReporter({
|
|
707
|
+
verbose: hasFlag(argv, "--verbose"),
|
|
708
|
+
quiet: hasFlag(argv, "--quiet"),
|
|
926
709
|
});
|
|
927
|
-
if (!
|
|
928
|
-
|
|
929
|
-
|
|
710
|
+
if (!config.dryRun
|
|
711
|
+
&& !config.paths.baseModel
|
|
712
|
+
&& !request.hyperparameters.base_model_revision) {
|
|
713
|
+
const prefetch = await prefetchBaseModel({
|
|
714
|
+
request,
|
|
715
|
+
config,
|
|
716
|
+
reporter,
|
|
717
|
+
});
|
|
718
|
+
if (!prefetch.snapshot_revision) {
|
|
719
|
+
throw new Error("Base-model prefetch did not return an immutable snapshot revision.");
|
|
930
720
|
}
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
const request = withParentModelArtifact(baseRequest, await parentModelSelectionFromArgv(argv, config));
|
|
937
|
-
assertLocalRunInputReady(request);
|
|
938
|
-
assertSupportedValidateShape(request, config);
|
|
939
|
-
assertEvaluationScoringReady(config);
|
|
940
|
-
if (hasFlag(argv, "--detach")) {
|
|
941
|
-
const detached = await launchDetachedRun({ argv, request, config });
|
|
942
|
-
const configPath = readOption(argv, "--config");
|
|
943
|
-
const configSuffix = configPath ? ` --config ${JSON.stringify(resolve(configPath))}` : "";
|
|
944
|
-
printJson({
|
|
945
|
-
status: "queued",
|
|
946
|
-
detached: true,
|
|
947
|
-
run_id: request.run_id,
|
|
948
|
-
behavior_spec_id: request.behavior_spec_id,
|
|
949
|
-
pid: detached.pid,
|
|
950
|
-
log_path: detached.logPath,
|
|
951
|
-
artifact_dir: detached.artifactDir,
|
|
952
|
-
next: {
|
|
953
|
-
watch: `tt-local runs watch ${request.run_id}${configSuffix}`,
|
|
954
|
-
cancel: `tt-local runs cancel ${request.run_id}${configSuffix}`,
|
|
721
|
+
request = fineTuneRunRequestSchema.parse({
|
|
722
|
+
...request,
|
|
723
|
+
hyperparameters: {
|
|
724
|
+
...request.hyperparameters,
|
|
725
|
+
base_model_revision: prefetch.snapshot_revision,
|
|
955
726
|
},
|
|
956
727
|
});
|
|
957
|
-
return;
|
|
958
728
|
}
|
|
959
|
-
const
|
|
960
|
-
const result = await runLocalFineTuneStage({
|
|
729
|
+
const result = await runLocalFineTune({
|
|
961
730
|
request,
|
|
962
731
|
config,
|
|
963
|
-
|
|
964
|
-
force: hasFlag(argv, "--force"),
|
|
965
|
-
modelArtifact: readOption(argv, "--model-artifact"),
|
|
966
|
-
reporter: createConsoleReporter({
|
|
967
|
-
verbose: hasFlag(argv, "--verbose"),
|
|
968
|
-
quiet: hasFlag(argv, "--quiet"),
|
|
969
|
-
}),
|
|
732
|
+
reporter,
|
|
970
733
|
});
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
report_path: result.reportPath,
|
|
979
|
-
artifact_dir: result.artifactDir,
|
|
734
|
+
printJson({
|
|
735
|
+
status: result.report.status,
|
|
736
|
+
run_id: result.report.run_id,
|
|
737
|
+
behavior_spec_id: result.report.behavior_spec_id,
|
|
738
|
+
report_path: result.reportPath,
|
|
739
|
+
artifact_dir: result.artifactDir,
|
|
740
|
+
...(!config.dryRun ? {
|
|
980
741
|
model_id: `local-${result.report.run_id}`,
|
|
981
742
|
fine_tuned_model_id: result.report.fine_tuned_model_id,
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
});
|
|
988
|
-
}
|
|
989
|
-
else {
|
|
990
|
-
printJson({
|
|
991
|
-
status: "stage_completed",
|
|
992
|
-
stage: result.stage,
|
|
993
|
-
input_kind: input.kind,
|
|
994
|
-
run_id: result.request.run_id,
|
|
995
|
-
behavior_spec_id: result.request.behavior_spec_id,
|
|
996
|
-
artifact_dir: result.artifactDir,
|
|
997
|
-
parent_model_artifact: result.request.hyperparameters.parent_model_artifact ?? null,
|
|
998
|
-
artifacts: result.artifacts,
|
|
999
|
-
});
|
|
1000
|
-
}
|
|
1001
|
-
return;
|
|
1002
|
-
}
|
|
1003
|
-
if (command === "label") {
|
|
1004
|
-
const sourcePath = cli.positionals[0];
|
|
1005
|
-
if (!sourcePath)
|
|
1006
|
-
throw new Error("label requires <input.jsonl|input.csv>");
|
|
1007
|
-
const configInput = await configFromArgv(argv);
|
|
1008
|
-
const config = localRunnerConfigSchema.parse({
|
|
1009
|
-
...configInput,
|
|
1010
|
-
dryRun: hasFlag(argv, "--dry-run") ? true : configInput.dryRun,
|
|
743
|
+
} : {}),
|
|
744
|
+
training_log: result.report.training.log_uri,
|
|
745
|
+
baseline_eval: result.report.artifact_uris.baseline_eval,
|
|
746
|
+
candidate_eval: result.report.artifact_uris.candidate_eval,
|
|
747
|
+
comparison: result.report.comparison,
|
|
1011
748
|
});
|
|
1012
|
-
const format = readOption(argv, "--format");
|
|
1013
|
-
if (format !== undefined && format !== "jsonl" && format !== "csv") {
|
|
1014
|
-
throw new Error(`--format must be jsonl or csv, got: ${format}`);
|
|
1015
|
-
}
|
|
1016
|
-
let systemMessage = readOption(argv, "--system-prompt");
|
|
1017
|
-
if (!systemMessage) {
|
|
1018
|
-
const specPath = resolve(readOption(argv, "--spec") ?? DEFAULT_LOCAL_SPEC_PATH);
|
|
1019
|
-
let raw;
|
|
1020
|
-
try {
|
|
1021
|
-
raw = await readFile(specPath, "utf8");
|
|
1022
|
-
}
|
|
1023
|
-
catch {
|
|
1024
|
-
throw new Error(`Cannot read spec at ${specPath}. Pass --spec <tunedtensor.json> or --system-prompt "..." so the teacher labels under the model's instructions.`);
|
|
1025
|
-
}
|
|
1026
|
-
systemMessage = buildSystemMessage(localBehaviorSpecFileSchema.parse(JSON.parse(raw)));
|
|
1027
|
-
}
|
|
1028
|
-
const result = await runLocalLabelingJob({
|
|
1029
|
-
sourcePath: resolve(sourcePath),
|
|
1030
|
-
format,
|
|
1031
|
-
inputColumn: readOption(argv, "--input-column"),
|
|
1032
|
-
systemMessage,
|
|
1033
|
-
config,
|
|
1034
|
-
outputPath: readOption(argv, "--output"),
|
|
1035
|
-
model: readOption(argv, "--model"),
|
|
1036
|
-
reporter: createConsoleReporter({
|
|
1037
|
-
verbose: hasFlag(argv, "--verbose"),
|
|
1038
|
-
quiet: hasFlag(argv, "--quiet"),
|
|
1039
|
-
}),
|
|
1040
|
-
});
|
|
1041
|
-
printJson(result.report);
|
|
1042
|
-
if (result.report.status === "failed")
|
|
1043
|
-
process.exitCode = 1;
|
|
1044
749
|
return;
|
|
1045
750
|
}
|
|
1046
751
|
if (command === "serve") {
|
|
1047
752
|
const config = await configFromArgv(argv);
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
await new Promise((resolveStop) => {
|
|
1053
|
-
const stop = () => {
|
|
1054
|
-
dashboard.close().then(resolveStop, resolveStop);
|
|
1055
|
-
};
|
|
1056
|
-
process.once("SIGINT", stop);
|
|
1057
|
-
process.once("SIGTERM", stop);
|
|
753
|
+
await serveStoredModelFromCli({
|
|
754
|
+
argv,
|
|
755
|
+
modelId: cli.positionals[0],
|
|
756
|
+
config,
|
|
1058
757
|
});
|
|
1059
758
|
return;
|
|
1060
759
|
}
|
|
@@ -1093,42 +792,14 @@ async function main(argv) {
|
|
|
1093
792
|
]);
|
|
1094
793
|
return printJson(compareRuns(reportA, reportB));
|
|
1095
794
|
}
|
|
1096
|
-
if (subcommand === "cancel") {
|
|
1097
|
-
const id = cli.positionals[0];
|
|
1098
|
-
if (!id)
|
|
1099
|
-
throw new Error("runs cancel requires <run-id>");
|
|
1100
|
-
await store.cancelRun(id);
|
|
1101
|
-
const run = await store.getRun(id);
|
|
1102
|
-
return printJson({ ok: true, run_id: run.id, status: run.status, current_stage: run.current_stage });
|
|
1103
|
-
}
|
|
1104
|
-
if (subcommand === "watch") {
|
|
1105
|
-
const id = cli.positionals[0];
|
|
1106
|
-
if (!id)
|
|
1107
|
-
throw new Error("runs watch requires <run-id>");
|
|
1108
|
-
const printed = new Set();
|
|
1109
|
-
while (true) {
|
|
1110
|
-
const events = await store.getRunEvents(id);
|
|
1111
|
-
for (const event of events) {
|
|
1112
|
-
if (printed.has(event.id))
|
|
1113
|
-
continue;
|
|
1114
|
-
printed.add(event.id);
|
|
1115
|
-
console.log(`${event.occurred_at} ${event.stage} ${event.status}: ${event.message}`);
|
|
1116
|
-
}
|
|
1117
|
-
const run = await store.getRun(id);
|
|
1118
|
-
if (isTerminalRunState(run))
|
|
1119
|
-
return;
|
|
1120
|
-
await sleep(1000);
|
|
1121
|
-
}
|
|
1122
|
-
}
|
|
1123
|
-
if (subcommand === "reconcile") {
|
|
1124
|
-
await store.rebuildIndexes();
|
|
1125
|
-
return printJson({ ok: true });
|
|
1126
|
-
}
|
|
1127
795
|
throw new Error(`Unknown runs command: ${subcommand}`);
|
|
1128
796
|
}
|
|
1129
797
|
if (command === "models") {
|
|
1130
798
|
const subcommand = cli.subcommand;
|
|
1131
|
-
const
|
|
799
|
+
const modelInputPath = subcommand === "prefetch" || subcommand === "verify-base"
|
|
800
|
+
? resolve(cli.positionals[0] ?? DEFAULT_LOCAL_SPEC_PATH)
|
|
801
|
+
: undefined;
|
|
802
|
+
const config = await configFromArgv(argv, modelInputPath);
|
|
1132
803
|
const store = createLocalStore(config.storeRoot);
|
|
1133
804
|
if (subcommand === "list")
|
|
1134
805
|
return printJson(await store.listModels());
|
|
@@ -1150,16 +821,7 @@ async function main(argv) {
|
|
|
1150
821
|
return printJson({ ok: true, model, ...verified });
|
|
1151
822
|
}
|
|
1152
823
|
if (subcommand === "prefetch" || subcommand === "verify-base") {
|
|
1153
|
-
const
|
|
1154
|
-
const input = await loadLocalRunInput(inputPath, {
|
|
1155
|
-
userId: readOption(argv, "--user-id"),
|
|
1156
|
-
runNumber: readNumberOption(argv, "--run-number"),
|
|
1157
|
-
});
|
|
1158
|
-
if (!hasFlag(argv, "--quiet")) {
|
|
1159
|
-
for (const warning of input.warnings) {
|
|
1160
|
-
process.stderr.write(`[tt-local] warning: ${warning}\n`);
|
|
1161
|
-
}
|
|
1162
|
-
}
|
|
824
|
+
const input = await loadCliBehaviorSpec(modelInputPath);
|
|
1163
825
|
const report = await prefetchBaseModel({
|
|
1164
826
|
request: input.request,
|
|
1165
827
|
config,
|
|
@@ -1171,7 +833,6 @@ async function main(argv) {
|
|
|
1171
833
|
});
|
|
1172
834
|
return printJson({
|
|
1173
835
|
...report,
|
|
1174
|
-
input_kind: input.kind,
|
|
1175
836
|
input_path: input.path,
|
|
1176
837
|
});
|
|
1177
838
|
}
|
|
@@ -1179,95 +840,11 @@ async function main(argv) {
|
|
|
1179
840
|
const id = cli.positionals[0];
|
|
1180
841
|
if (!id)
|
|
1181
842
|
throw new Error("models serve requires <model-id>");
|
|
1182
|
-
|
|
1183
|
-
const verified = await verifyStoredModel(model, { requireServable: true });
|
|
1184
|
-
const launch = buildLocalModelServerLaunch({
|
|
1185
|
-
model,
|
|
1186
|
-
config,
|
|
1187
|
-
options: {
|
|
1188
|
-
host: readOption(argv, "--host"),
|
|
1189
|
-
port: readNumberOption(argv, "--port"),
|
|
1190
|
-
device: modelServeDevice(argv),
|
|
1191
|
-
maxTokens: readNumberOption(argv, "--max-tokens"),
|
|
1192
|
-
temperature: readNumberOption(argv, "--temperature"),
|
|
1193
|
-
topP: readNumberOption(argv, "--top-p"),
|
|
1194
|
-
maxConcurrentRequests: readNumberOption(argv, "--max-concurrent-requests"),
|
|
1195
|
-
systemPrompt: await modelSystemPrompt({ argv, model, store }),
|
|
1196
|
-
allowRemote: hasFlag(argv, "--allow-remote"),
|
|
1197
|
-
apiKeyEnv: readOption(argv, "--api-key-env"),
|
|
1198
|
-
baseModelRevision: (() => {
|
|
1199
|
-
const contract = verified.contract;
|
|
1200
|
-
return typeof contract.base_model_revision === "string" ? contract.base_model_revision : undefined;
|
|
1201
|
-
})(),
|
|
1202
|
-
baseModelArtifactUri: (() => {
|
|
1203
|
-
const contract = verified.contract;
|
|
1204
|
-
return typeof contract.base_model_artifact_uri === "string" ? contract.base_model_artifact_uri : undefined;
|
|
1205
|
-
})(),
|
|
1206
|
-
},
|
|
1207
|
-
});
|
|
1208
|
-
if (hasFlag(argv, "--print-command")) {
|
|
1209
|
-
return printJson({
|
|
1210
|
-
ok: true,
|
|
1211
|
-
model_id: model.id,
|
|
1212
|
-
url: launch.url,
|
|
1213
|
-
command: launch.displayCommand,
|
|
1214
|
-
artifact_path: launch.artifactPath,
|
|
1215
|
-
manifest_path: verified.manifest_path,
|
|
1216
|
-
integrity: verified.integrity,
|
|
1217
|
-
});
|
|
1218
|
-
}
|
|
1219
|
-
process.stderr.write(`[tt-local] verified ${verified.integrity.checked} artifact file(s)\n`);
|
|
1220
|
-
process.stderr.write(`[tt-local] model API: ${launch.url}\n`);
|
|
1221
|
-
await serveLocalModel(launch);
|
|
843
|
+
await serveStoredModelFromCli({ argv, modelId: id, config });
|
|
1222
844
|
return;
|
|
1223
845
|
}
|
|
1224
846
|
throw new Error(`Unknown models command: ${subcommand}`);
|
|
1225
847
|
}
|
|
1226
|
-
if (command === "specs") {
|
|
1227
|
-
const subcommand = cli.subcommand;
|
|
1228
|
-
const config = await configFromArgv(argv);
|
|
1229
|
-
const store = createLocalStore(config.storeRoot);
|
|
1230
|
-
if (subcommand === "list")
|
|
1231
|
-
return printJson(await store.listSpecs());
|
|
1232
|
-
if (subcommand === "get") {
|
|
1233
|
-
const id = cli.positionals[0];
|
|
1234
|
-
if (!id)
|
|
1235
|
-
throw new Error("specs get requires <spec-id>");
|
|
1236
|
-
return printJson(await store.getSpec(id));
|
|
1237
|
-
}
|
|
1238
|
-
if (subcommand === "import") {
|
|
1239
|
-
const path = cli.positionals[0];
|
|
1240
|
-
if (!path)
|
|
1241
|
-
throw new Error("specs import requires <spec-or-request.json>");
|
|
1242
|
-
const input = JSON.parse(await readFile(resolve(path), "utf8"));
|
|
1243
|
-
const request = fineTuneRunRequestSchema.safeParse(input);
|
|
1244
|
-
if (request.success)
|
|
1245
|
-
return printJson(await store.importSpec(request.data.behavior_spec_id, request.data.spec_snapshot));
|
|
1246
|
-
const localSpec = localBehaviorSpecFileSchema.safeParse(input);
|
|
1247
|
-
if (localSpec.success) {
|
|
1248
|
-
const localRequest = runRequestFromLocalSpec(localSpec.data, {
|
|
1249
|
-
userId: readOption(argv, "--user-id"),
|
|
1250
|
-
runNumber: readNumberOption(argv, "--run-number"),
|
|
1251
|
-
});
|
|
1252
|
-
return printJson(await store.importSpec(localRequest.behavior_spec_id, localRequest.spec_snapshot));
|
|
1253
|
-
}
|
|
1254
|
-
const id = readOption(argv, "--id");
|
|
1255
|
-
if (!id)
|
|
1256
|
-
throw new Error("specs import requires --id when importing a raw spec snapshot");
|
|
1257
|
-
return printJson(await store.importSpec(id, specSnapshotSchema.parse(input)));
|
|
1258
|
-
}
|
|
1259
|
-
throw new Error(`Unknown specs command: ${subcommand}`);
|
|
1260
|
-
}
|
|
1261
|
-
if (command === "store") {
|
|
1262
|
-
const subcommand = cli.subcommand;
|
|
1263
|
-
const config = await configFromArgv(argv);
|
|
1264
|
-
const store = createLocalStore(config.storeRoot);
|
|
1265
|
-
if (subcommand === "rebuild-index") {
|
|
1266
|
-
await store.rebuildIndexes();
|
|
1267
|
-
return printJson({ ok: true });
|
|
1268
|
-
}
|
|
1269
|
-
throw new Error(`Unknown store command: ${subcommand ?? ""}`);
|
|
1270
|
-
}
|
|
1271
848
|
console.error(`Unknown command: ${command}`);
|
|
1272
849
|
process.exitCode = 1;
|
|
1273
850
|
}
|