@tuned-tensor/cli 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -28
- package/dist/{chunk-KRKISPHG.js → chunk-U5HNS367.js} +319 -40
- package/dist/chunk-U5HNS367.js.map +1 -0
- package/dist/index.js +290 -341
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +26 -39
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/aws-gpu.md +28 -27
- package/docs/local-runtime/serving.md +2 -2
- package/docs/local-runtime/spark.md +12 -17
- package/docs/spec-architecture.html +18 -18
- package/docs/spec-workflow.md +27 -6
- package/examples/single-spec/adapter/tunedtensor.json +49 -0
- package/examples/single-spec/foundation/tunedtensor.json +31 -0
- package/package.json +4 -3
- package/training/adapter/src/serve.py +6 -0
- package/dist/chunk-KRKISPHG.js.map +0 -1
package/README.md
CHANGED
|
@@ -288,29 +288,32 @@ expected output for every example. Treat those metrics as an end-to-end
|
|
|
288
288
|
execution and overfit check—not held-out capability or multi-GPU evidence when
|
|
289
289
|
no validation corpus is configured.
|
|
290
290
|
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
291
|
+
`tunedtensor.json` contains behavior, training settings, optional `evaluation`,
|
|
292
|
+
`runtime`, and advanced `pipeline` sections. Pipelines use the embedded recipe
|
|
293
|
+
or derive it from the saved spec. Only explicit `--file` loads a separate recipe;
|
|
294
|
+
it cannot override a conflicting embedded recipe. Foundation recipes must match
|
|
295
|
+
the spec's training settings.
|
|
294
296
|
|
|
295
297
|
```bash
|
|
296
|
-
tt pipeline
|
|
297
|
-
tt pipeline
|
|
298
|
-
tt pipeline
|
|
299
|
-
tt
|
|
300
|
-
|
|
301
|
-
tt
|
|
302
|
-
|
|
303
|
-
tt
|
|
304
|
-
tt --json pipeline run --spec tunedtensor.json --dry-run
|
|
305
|
-
tt pipeline run --spec tunedtensor.json --resume /absolute/path/to/foundation-run
|
|
298
|
+
tt pipeline validate --spec tunedtensor.json
|
|
299
|
+
tt --json pipeline plan --spec tunedtensor.json
|
|
300
|
+
tt pipeline run --spec tunedtensor.json --dry-run
|
|
301
|
+
tt pipeline run --spec tunedtensor.json
|
|
302
|
+
# Optional: embed the current derived recipe for customization
|
|
303
|
+
tt pipeline init --spec tunedtensor.json
|
|
304
|
+
# Existing projects: consolidate sidecars, retaining .bak copies
|
|
305
|
+
tt pipeline migrate --spec tunedtensor.json
|
|
306
306
|
```
|
|
307
307
|
|
|
308
|
-
|
|
309
|
-
|
|
308
|
+
Relative runtime paths resolve beside the spec. New project state uses
|
|
309
|
+
`.tuned-tensor/store` and `.tuned-tensor/artifacts`. Legacy `local-runner.json`
|
|
310
|
+
remains readable with a warning; migrate it before adding inline runtime or
|
|
311
|
+
evaluation settings. See [single-file examples](examples/single-spec) and the
|
|
312
|
+
[spec workflow](docs/spec-workflow.md).
|
|
310
313
|
|
|
311
314
|
`--only` and `--skip` preserve dependency safety: a selected step cannot refer
|
|
312
|
-
to an omitted predecessor. Keep step targets local and configure `gpu` in
|
|
313
|
-
`
|
|
315
|
+
to an omitted predecessor. Keep step targets local and configure `runtime.gpu` in
|
|
316
|
+
`tunedtensor.json` to execute GPU work on your AWS instance. Adapter and
|
|
314
317
|
foundation workflows use the same local pipeline. The TT agent may describe,
|
|
315
318
|
validate, and prepare a sealed pipeline action; only deterministic `/approve`
|
|
316
319
|
handling can execute it.
|
|
@@ -389,18 +392,20 @@ tt init \
|
|
|
389
392
|
|
|
390
393
|
Edit the generated `tunedtensor.json`, replacing the placeholder system prompt and both placeholder examples.
|
|
391
394
|
For a local GPU, `tt hardware` reports this machine's training and serving
|
|
392
|
-
capacity. For an AWS GPU,
|
|
393
|
-
|
|
395
|
+
capacity. For an AWS GPU, add the following `runtime` section to your
|
|
396
|
+
`tunedtensor.json`, replacing the instance, region, profile and SSH settings with your own:
|
|
394
397
|
|
|
395
398
|
```json
|
|
396
399
|
{
|
|
397
|
-
"
|
|
398
|
-
"
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
400
|
+
"runtime": {
|
|
401
|
+
"gpu": {
|
|
402
|
+
"provider": "aws",
|
|
403
|
+
"profile": "research",
|
|
404
|
+
"region": "eu-west-1",
|
|
405
|
+
"instanceId": "i-0123456789abcdef0",
|
|
406
|
+
"user": "ubuntu",
|
|
407
|
+
"identityFile": "~/.ssh/research-gpu.pem"
|
|
408
|
+
}
|
|
404
409
|
}
|
|
405
410
|
}
|
|
406
411
|
```
|
|
@@ -419,7 +424,7 @@ tt pipeline run --spec tunedtensor.json --dry-run
|
|
|
419
424
|
tt pipeline run --spec tunedtensor.json
|
|
420
425
|
```
|
|
421
426
|
|
|
422
|
-
These commands
|
|
427
|
+
These commands read runtime and evaluation settings from the core spec. The preview needs
|
|
423
428
|
no GPU or AWS connection. Results return to the local run and model stores:
|
|
424
429
|
|
|
425
430
|
```bash
|
|
@@ -451,7 +456,7 @@ Activation is optional and requires the run to pass a configured
|
|
|
451
456
|
`generalRegression` gate. Without that suite, `tt models activate` fails
|
|
452
457
|
closed. `tt serve active` also fails closed if nothing is activated — it does
|
|
453
458
|
not silently serve the protected base model. Once activated, use
|
|
454
|
-
`tt serve active
|
|
459
|
+
`tt serve active`.
|
|
455
460
|
|
|
456
461
|
`tt serve base` does not automatically inject the adjacent project spec.
|
|
457
462
|
Pass `--spec tunedtensor.json` when the server should enforce its instructions.
|
|
@@ -836,6 +836,7 @@ async function assertUsableModelArtifact(uri) {
|
|
|
836
836
|
|
|
837
837
|
// src/local-runtime/contracts.ts
|
|
838
838
|
import { z } from "zod";
|
|
839
|
+
import { pipelineDocumentSchema, isFoundationPipeline, parsePipeline } from "@tuned-tensor/pipeline-contract";
|
|
839
840
|
|
|
840
841
|
// src/paths.ts
|
|
841
842
|
import { existsSync } from "fs";
|
|
@@ -946,12 +947,6 @@ var foundationHyperparametersSchema = z.object({
|
|
|
946
947
|
keep_checkpoints: z.number().int().min(1).max(100).optional(),
|
|
947
948
|
log_interval_steps: z.number().int().min(1).max(1e6).optional()
|
|
948
949
|
}).strict();
|
|
949
|
-
var localFoundationSpecFileSchema = behaviorSpecSchema.extend({
|
|
950
|
-
engine: z.literal("foundation"),
|
|
951
|
-
id: z.string().uuid().optional(),
|
|
952
|
-
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
953
|
-
foundation: foundationHyperparametersSchema
|
|
954
|
-
});
|
|
955
950
|
var datasetPrebuiltSchema = z.object({
|
|
956
951
|
training: z.string().min(1),
|
|
957
952
|
validation: z.string().min(1).optional(),
|
|
@@ -996,16 +991,6 @@ var fineTuneRunRequestSchema = z.object({
|
|
|
996
991
|
});
|
|
997
992
|
}
|
|
998
993
|
});
|
|
999
|
-
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
1000
|
-
engine: z.literal("adapter").optional(),
|
|
1001
|
-
id: z.string().uuid().optional(),
|
|
1002
|
-
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
1003
|
-
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
1004
|
-
}).strict();
|
|
1005
|
-
var localBehaviorSpecFileSchema = z.union([
|
|
1006
|
-
localFoundationSpecFileSchema,
|
|
1007
|
-
localAdapterSpecFileSchema
|
|
1008
|
-
]);
|
|
1009
994
|
var evalExampleResultSchema = z.object({
|
|
1010
995
|
prompt: z.string(),
|
|
1011
996
|
expected: z.string(),
|
|
@@ -1196,6 +1181,42 @@ var localRunnerConfigSchema = z.object({
|
|
|
1196
1181
|
baselineCache: true
|
|
1197
1182
|
})
|
|
1198
1183
|
}).strict();
|
|
1184
|
+
var projectRuntimeSchema = z.object({
|
|
1185
|
+
gpu: localRunnerConfigSchema.shape.gpu,
|
|
1186
|
+
artifactRoot: z.string().min(1).optional(),
|
|
1187
|
+
storeRoot: z.string().min(1).optional(),
|
|
1188
|
+
paths: z.object({ baseModel: z.string().min(1).optional(), modelCache: z.string().min(1).optional() }).strict().optional()
|
|
1189
|
+
}).strict();
|
|
1190
|
+
var projectWorkflowFields = {
|
|
1191
|
+
evaluation: evaluationConfigSchema.optional(),
|
|
1192
|
+
runtime: projectRuntimeSchema.optional(),
|
|
1193
|
+
pipeline: pipelineDocumentSchema.optional()
|
|
1194
|
+
};
|
|
1195
|
+
var localFoundationSpecFileSchema = behaviorSpecSchema.extend({
|
|
1196
|
+
...projectWorkflowFields,
|
|
1197
|
+
engine: z.literal("foundation"),
|
|
1198
|
+
id: z.string().uuid().optional(),
|
|
1199
|
+
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
1200
|
+
foundation: foundationHyperparametersSchema
|
|
1201
|
+
});
|
|
1202
|
+
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
1203
|
+
...projectWorkflowFields,
|
|
1204
|
+
engine: z.literal("adapter").optional(),
|
|
1205
|
+
id: z.string().uuid().optional(),
|
|
1206
|
+
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
1207
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
1208
|
+
}).strict();
|
|
1209
|
+
var localBehaviorSpecFileSchema = z.union([
|
|
1210
|
+
localFoundationSpecFileSchema,
|
|
1211
|
+
localAdapterSpecFileSchema
|
|
1212
|
+
]).superRefine((spec, ctx) => {
|
|
1213
|
+
if (spec.pipeline && isFoundationPipeline(parsePipeline(spec.pipeline)) !== (spec.engine === "foundation")) {
|
|
1214
|
+
ctx.addIssue({ code: "custom", path: ["pipeline"], message: "Pipeline engine must match the spec engine." });
|
|
1215
|
+
}
|
|
1216
|
+
if (spec.engine === "foundation" && (spec.evaluation || spec.runtime?.storeRoot || spec.runtime?.paths)) {
|
|
1217
|
+
ctx.addIssue({ code: "custom", path: ["runtime"], message: "Foundation uses foundation settings for evaluation and checkpoints; runtime.artifactRoot and runtime.gpu are supported." });
|
|
1218
|
+
}
|
|
1219
|
+
});
|
|
1199
1220
|
function isFoundationSpecFile(spec) {
|
|
1200
1221
|
return spec.engine === "foundation";
|
|
1201
1222
|
}
|
|
@@ -1233,6 +1254,18 @@ function resolveLocalRunInputPaths(raw, inputPath) {
|
|
|
1233
1254
|
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
1234
1255
|
}
|
|
1235
1256
|
}
|
|
1257
|
+
const runtime = value.runtime;
|
|
1258
|
+
if (runtime && typeof runtime === "object") {
|
|
1259
|
+
for (const key of ["artifactRoot", "storeRoot"]) if (runtime[key] !== void 0) runtime[key] = resolveLocalReference(runtime[key], baseDirectory);
|
|
1260
|
+
const gpu = runtime.gpu;
|
|
1261
|
+
if (gpu?.identityFile !== void 0) gpu.identityFile = resolveLocalReference(gpu.identityFile, baseDirectory);
|
|
1262
|
+
const paths = runtime.paths;
|
|
1263
|
+
if (paths && typeof paths === "object") {
|
|
1264
|
+
for (const key of ["baseModel", "modelCache"]) if (paths[key] !== void 0) paths[key] = resolveLocalReference(paths[key], baseDirectory);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
const regression = value.evaluation?.generalRegression;
|
|
1268
|
+
if (regression?.dataset !== void 0) regression.dataset = resolveLocalReference(regression.dataset, baseDirectory);
|
|
1236
1269
|
const foundation = value.foundation;
|
|
1237
1270
|
if (foundation && typeof foundation === "object" && !Array.isArray(foundation)) {
|
|
1238
1271
|
const fields = foundation;
|
|
@@ -1385,6 +1418,9 @@ function runRequestFromLocalSpec(spec, options = {}) {
|
|
|
1385
1418
|
const {
|
|
1386
1419
|
id,
|
|
1387
1420
|
engine: _engine,
|
|
1421
|
+
evaluation: _evaluation,
|
|
1422
|
+
runtime: _runtime,
|
|
1423
|
+
pipeline: _pipeline,
|
|
1388
1424
|
hyperparameters,
|
|
1389
1425
|
dataset_prebuilt: datasetPrebuilt,
|
|
1390
1426
|
...specSnapshot
|
|
@@ -1489,6 +1525,199 @@ function validateBehaviorSpec(input) {
|
|
|
1489
1525
|
return { valid: errors.length === 0, errors, warnings };
|
|
1490
1526
|
}
|
|
1491
1527
|
|
|
1528
|
+
// src/pipeline.ts
|
|
1529
|
+
import {
|
|
1530
|
+
canonicalJson,
|
|
1531
|
+
isFoundationPipeline as isFoundationPipeline2,
|
|
1532
|
+
canonicalPipeline as canonicalPortablePipeline,
|
|
1533
|
+
parsePipeline as parsePipeline2,
|
|
1534
|
+
pipelineDocumentSchema as pipelineDocumentSchema2
|
|
1535
|
+
} from "@tuned-tensor/pipeline-contract";
|
|
1536
|
+
import {
|
|
1537
|
+
PIPELINE_DOCUMENT_VERSION,
|
|
1538
|
+
PIPELINE_EVALUATORS,
|
|
1539
|
+
PIPELINE_MAX_STEPS,
|
|
1540
|
+
PIPELINE_MAX_TRAIN_STEPS,
|
|
1541
|
+
PIPELINE_STEP_CAPABILITIES,
|
|
1542
|
+
PIPELINE_STEP_USES,
|
|
1543
|
+
PIPELINE_TARGETS,
|
|
1544
|
+
canonicalFoundationPipeline as canonicalFoundationPipeline2,
|
|
1545
|
+
canonicalJson as canonicalJson2,
|
|
1546
|
+
isFoundationPipeline as isFoundationPipeline3,
|
|
1547
|
+
parsePipeline as parsePipeline3,
|
|
1548
|
+
pipelineDocumentSchema as pipelineDocumentSchema3,
|
|
1549
|
+
pipelineHash
|
|
1550
|
+
} from "@tuned-tensor/pipeline-contract";
|
|
1551
|
+
function stepReferences(step) {
|
|
1552
|
+
if (step.uses === "evaluate") return step.with.model === "base" ? [] : [step.with.model];
|
|
1553
|
+
if (step.uses === "compare") return [step.with.before, step.with.after];
|
|
1554
|
+
if (step.uses === "pretrain") return [step.with.tokenizer];
|
|
1555
|
+
if (step.uses === "finetune" || step.uses === "rl") return [step.with.model];
|
|
1556
|
+
return [];
|
|
1557
|
+
}
|
|
1558
|
+
function pipelineFromFoundationHyperparameters(name, hp) {
|
|
1559
|
+
const lastTrainId = hp.rl_steps > 0 ? "rl" : "sft";
|
|
1560
|
+
const steps = [
|
|
1561
|
+
{
|
|
1562
|
+
id: "tokenize",
|
|
1563
|
+
uses: "tokenize",
|
|
1564
|
+
target: "local",
|
|
1565
|
+
with: { vocabSize: hp.vocab_size, maxChars: hp.max_chars }
|
|
1566
|
+
},
|
|
1567
|
+
{
|
|
1568
|
+
id: "pretrain",
|
|
1569
|
+
uses: "pretrain",
|
|
1570
|
+
target: "local",
|
|
1571
|
+
with: {
|
|
1572
|
+
tokenizer: { from: "tokenize.tokenizer" },
|
|
1573
|
+
depth: hp.depth,
|
|
1574
|
+
steps: hp.pretrain_steps,
|
|
1575
|
+
batchSize: hp.batch_size,
|
|
1576
|
+
sequenceLength: hp.sequence_length,
|
|
1577
|
+
nprocPerNode: hp.nproc_per_node
|
|
1578
|
+
}
|
|
1579
|
+
},
|
|
1580
|
+
{
|
|
1581
|
+
id: "bpb",
|
|
1582
|
+
uses: "evaluate",
|
|
1583
|
+
target: "local",
|
|
1584
|
+
with: { model: { from: "pretrain.model" }, evaluator: "bpb" }
|
|
1585
|
+
},
|
|
1586
|
+
{
|
|
1587
|
+
id: "sft",
|
|
1588
|
+
uses: "finetune",
|
|
1589
|
+
target: "local",
|
|
1590
|
+
with: {
|
|
1591
|
+
model: { from: "pretrain.model" },
|
|
1592
|
+
steps: hp.finetune_steps,
|
|
1593
|
+
batchSize: hp.batch_size
|
|
1594
|
+
}
|
|
1595
|
+
},
|
|
1596
|
+
{
|
|
1597
|
+
id: "chat",
|
|
1598
|
+
uses: "evaluate",
|
|
1599
|
+
target: "local",
|
|
1600
|
+
with: { model: { from: "sft.model" }, evaluator: "chat" }
|
|
1601
|
+
}
|
|
1602
|
+
];
|
|
1603
|
+
if (hp.rl_steps > 0) {
|
|
1604
|
+
steps.push(
|
|
1605
|
+
{
|
|
1606
|
+
id: "rl",
|
|
1607
|
+
uses: "rl",
|
|
1608
|
+
target: "local",
|
|
1609
|
+
with: { model: { from: "sft.model" }, steps: hp.rl_steps }
|
|
1610
|
+
},
|
|
1611
|
+
{
|
|
1612
|
+
id: "chat_rl",
|
|
1613
|
+
uses: "evaluate",
|
|
1614
|
+
target: "local",
|
|
1615
|
+
with: { model: { from: "rl.model" }, evaluator: "chat" }
|
|
1616
|
+
}
|
|
1617
|
+
);
|
|
1618
|
+
}
|
|
1619
|
+
steps.push({
|
|
1620
|
+
id: "infer",
|
|
1621
|
+
uses: "evaluate",
|
|
1622
|
+
target: "local",
|
|
1623
|
+
with: { model: { from: `${lastTrainId}.model` }, evaluator: "inference" }
|
|
1624
|
+
});
|
|
1625
|
+
return {
|
|
1626
|
+
version: 1,
|
|
1627
|
+
name,
|
|
1628
|
+
runtime: { engine: "foundation" },
|
|
1629
|
+
steps
|
|
1630
|
+
};
|
|
1631
|
+
}
|
|
1632
|
+
function producerId(reference) {
|
|
1633
|
+
return reference.from.slice(0, reference.from.lastIndexOf("."));
|
|
1634
|
+
}
|
|
1635
|
+
function validatePipeline(pipeline) {
|
|
1636
|
+
const structural = pipelineDocumentSchema2.safeParse(pipeline);
|
|
1637
|
+
if (!structural.success) {
|
|
1638
|
+
return structural.error.issues.map((issue) => {
|
|
1639
|
+
const path = issue.path.map(String).join(".");
|
|
1640
|
+
return path ? `${path}: ${issue.message}` : issue.message;
|
|
1641
|
+
});
|
|
1642
|
+
}
|
|
1643
|
+
try {
|
|
1644
|
+
parsePipeline2(pipeline);
|
|
1645
|
+
return [];
|
|
1646
|
+
} catch (error) {
|
|
1647
|
+
return [error instanceof Error ? error.message : String(error)];
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
function canonicalPipeline(target = "local") {
|
|
1651
|
+
return canonicalPortablePipeline(target);
|
|
1652
|
+
}
|
|
1653
|
+
function pipelineForRunInput(input, requested) {
|
|
1654
|
+
if (input.kind === "request") {
|
|
1655
|
+
assertLocalRunInputReady(input.request);
|
|
1656
|
+
} else {
|
|
1657
|
+
const validation = validateBehaviorSpec(input);
|
|
1658
|
+
if (!validation.valid) {
|
|
1659
|
+
throw new Error(`Invalid behavior spec:
|
|
1660
|
+
${validation.errors.join("\n")}`);
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
const foundation = input.kind === "foundation-spec";
|
|
1664
|
+
const embedded = input.kind === "request" ? void 0 : input.spec.pipeline;
|
|
1665
|
+
if (embedded && requested !== void 0 && canonicalJson(parsePipeline2(embedded)) !== canonicalJson(parsePipeline2(requested))) {
|
|
1666
|
+
throw new Error("Conflicting pipeline settings: update tunedtensor.json.pipeline before running a different recipe.");
|
|
1667
|
+
}
|
|
1668
|
+
const recipe = requested !== void 0 ? requested : embedded ?? (foundation ? pipelineFromFoundationHyperparameters(input.spec.name, input.spec.foundation) : canonicalPipeline("local"));
|
|
1669
|
+
const pipeline = parsePipeline2(recipe);
|
|
1670
|
+
if (isFoundationPipeline2(pipeline) !== foundation) {
|
|
1671
|
+
const recipeEngine = isFoundationPipeline2(pipeline) ? "a foundation" : "an adapter";
|
|
1672
|
+
const specEngine = foundation ? "a foundation" : "an adapter";
|
|
1673
|
+
throw new Error(`Pipeline is ${recipeEngine} recipe, but the behavior spec is ${specEngine} spec. Use a matching recipe or omit --file.`);
|
|
1674
|
+
}
|
|
1675
|
+
if (foundation) assertFoundationSettingsMatch(input.spec.foundation, pipeline);
|
|
1676
|
+
return pipeline;
|
|
1677
|
+
}
|
|
1678
|
+
function createExecutionPlan(pipeline, selection = {}) {
|
|
1679
|
+
const errors = validatePipeline(pipeline);
|
|
1680
|
+
if (errors.length) throw new Error(`Invalid pipeline:
|
|
1681
|
+
- ${errors.join("\n- ")}`);
|
|
1682
|
+
const normalized = parsePipeline2(pipeline);
|
|
1683
|
+
const allIds = new Set(normalized.steps.map((step) => step.id));
|
|
1684
|
+
for (const id of [...selection.only ?? [], ...selection.skip ?? []]) {
|
|
1685
|
+
if (!allIds.has(id)) throw new Error(`Unknown pipeline step: ${id}.`);
|
|
1686
|
+
}
|
|
1687
|
+
const only = selection.only?.length ? new Set(selection.only) : void 0;
|
|
1688
|
+
const skip = new Set(selection.skip ?? []);
|
|
1689
|
+
const steps = normalized.steps.filter((step) => (!only || only.has(step.id)) && !skip.has(step.id));
|
|
1690
|
+
if (!steps.length) throw new Error("Selection leaves no pipeline steps to run.");
|
|
1691
|
+
const retained = new Map(steps.map((step) => [step.id, step]));
|
|
1692
|
+
const resolved = steps.map((step) => {
|
|
1693
|
+
const transfers = [];
|
|
1694
|
+
for (const reference of stepReferences(step)) {
|
|
1695
|
+
const producer = retained.get(producerId(reference));
|
|
1696
|
+
if (!producer) throw new Error(`Step ${step.id} has a dependency on ${producerId(reference)}, which was omitted by --only or --skip.`);
|
|
1697
|
+
if (producer.target !== step.target) transfers.push({ from: reference.from, from_target: producer.target, to_target: step.target });
|
|
1698
|
+
}
|
|
1699
|
+
return { ...step, transfers };
|
|
1700
|
+
});
|
|
1701
|
+
return { version: 1, ...normalized.name ? { name: normalized.name } : {}, steps: resolved };
|
|
1702
|
+
}
|
|
1703
|
+
function assertFoundationSettingsMatch(hp, plan) {
|
|
1704
|
+
const settings = {
|
|
1705
|
+
tokenize: { vocabSize: hp.vocab_size, maxChars: hp.max_chars },
|
|
1706
|
+
pretrain: { depth: hp.depth, steps: hp.pretrain_steps, batchSize: hp.batch_size, sequenceLength: hp.sequence_length, nprocPerNode: hp.nproc_per_node },
|
|
1707
|
+
finetune: { steps: hp.finetune_steps, batchSize: hp.batch_size },
|
|
1708
|
+
rl: { steps: hp.rl_steps }
|
|
1709
|
+
};
|
|
1710
|
+
for (const step of plan.steps) {
|
|
1711
|
+
const fields = "with" in step ? step.with : void 0;
|
|
1712
|
+
for (const [key, expected] of Object.entries(settings[step.uses] ?? {})) {
|
|
1713
|
+
if (fields?.[key] !== void 0 && fields[key] !== expected) {
|
|
1714
|
+
throw new Error(`Pipeline step ${step.id}.${key} conflicts with the behavior spec. Edit tunedtensor.json and regenerate the recipe, or omit --file.`);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
if (step.uses === "rl" && hp.rl_steps === 0) throw new Error("The behavior spec disables RL; update foundation.rl_steps before adding an RL stage.");
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1492
1721
|
// src/local-runtime/dataset.ts
|
|
1493
1722
|
import { readFile as readFile3 } from "fs/promises";
|
|
1494
1723
|
async function loadNormalizedChatJsonl(path) {
|
|
@@ -3294,11 +3523,11 @@ function extractJsonObject(value) {
|
|
|
3294
3523
|
}
|
|
3295
3524
|
return null;
|
|
3296
3525
|
}
|
|
3297
|
-
function
|
|
3298
|
-
if (Array.isArray(value)) return value.map(
|
|
3526
|
+
function canonicalJson3(value) {
|
|
3527
|
+
if (Array.isArray(value)) return value.map(canonicalJson3);
|
|
3299
3528
|
if (value && typeof value === "object") {
|
|
3300
3529
|
return Object.fromEntries(
|
|
3301
|
-
Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key,
|
|
3530
|
+
Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalJson3(item)])
|
|
3302
3531
|
);
|
|
3303
3532
|
}
|
|
3304
3533
|
return value;
|
|
@@ -3324,7 +3553,7 @@ function scoreJsonFields(expected, actual, configuredFields) {
|
|
|
3324
3553
|
const schemaMatch = Boolean(actualJson) && expectedKeys.length === actualKeys.length && expectedKeys.every((key, index) => key === actualKeys[index]);
|
|
3325
3554
|
const fieldResults = {};
|
|
3326
3555
|
for (const field of fields) {
|
|
3327
|
-
fieldResults[field] = Object.prototype.hasOwnProperty.call(expectedJson, field) && Boolean(actualJson) && JSON.stringify(
|
|
3556
|
+
fieldResults[field] = Object.prototype.hasOwnProperty.call(expectedJson, field) && Boolean(actualJson) && JSON.stringify(canonicalJson3(expectedJson[field])) === JSON.stringify(canonicalJson3(actualJson?.[field]));
|
|
3328
3557
|
}
|
|
3329
3558
|
const correct = Object.values(fieldResults).filter(Boolean).length;
|
|
3330
3559
|
const score = fields.length > 0 ? correct / fields.length : 0;
|
|
@@ -4224,14 +4453,14 @@ function artifactPrefix(request) {
|
|
|
4224
4453
|
runId: request.run_id
|
|
4225
4454
|
});
|
|
4226
4455
|
}
|
|
4227
|
-
function
|
|
4456
|
+
function canonicalJson4(value) {
|
|
4228
4457
|
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
|
|
4229
|
-
if (Array.isArray(value)) return `[${value.map((entry) =>
|
|
4458
|
+
if (Array.isArray(value)) return `[${value.map((entry) => canonicalJson4(entry)).join(",")}]`;
|
|
4230
4459
|
const object = value;
|
|
4231
|
-
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${
|
|
4460
|
+
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson4(object[key])}`).join(",")}}`;
|
|
4232
4461
|
}
|
|
4233
4462
|
function hashJson(value) {
|
|
4234
|
-
return createHash7("sha256").update(
|
|
4463
|
+
return createHash7("sha256").update(canonicalJson4(value)).digest("hex");
|
|
4235
4464
|
}
|
|
4236
4465
|
var packageRoot2 = localRuntimePackageRoot(import.meta.url);
|
|
4237
4466
|
async function hashFileIfPresent(path) {
|
|
@@ -5294,6 +5523,16 @@ async function runLocalPipeline(input) {
|
|
|
5294
5523
|
store,
|
|
5295
5524
|
reporter: input.reporter
|
|
5296
5525
|
});
|
|
5526
|
+
await writeJsonAtomic(join10(artifacts.runDir, "resolved-workflow.json"), {
|
|
5527
|
+
spec: input.projectSpec ?? {
|
|
5528
|
+
...input.request.spec_snapshot,
|
|
5529
|
+
hyperparameters: input.request.hyperparameters,
|
|
5530
|
+
dataset_prebuilt: input.request.dataset_prebuilt
|
|
5531
|
+
},
|
|
5532
|
+
request: input.request,
|
|
5533
|
+
runtime: input.config,
|
|
5534
|
+
pipeline
|
|
5535
|
+
});
|
|
5297
5536
|
const outputs = {};
|
|
5298
5537
|
for (const step of pipeline.steps) {
|
|
5299
5538
|
await throwIfCancelled(store, input.request);
|
|
@@ -5381,6 +5620,32 @@ async function runLocalFineTune(input) {
|
|
|
5381
5620
|
};
|
|
5382
5621
|
}
|
|
5383
5622
|
|
|
5623
|
+
// src/local-runtime/project-workflow.ts
|
|
5624
|
+
import { existsSync as existsSync3 } from "fs";
|
|
5625
|
+
import { readFile as readFile9 } from "fs/promises";
|
|
5626
|
+
import { dirname as dirname10, join as join11, resolve as resolve14 } from "path";
|
|
5627
|
+
function configForSpec(spec, specPath) {
|
|
5628
|
+
return parseLocalRunnerConfig({
|
|
5629
|
+
storeRoot: process.env.TT_LOCAL_HOME?.trim() ? expandUserPath(process.env.TT_LOCAL_HOME.trim()) : DEFAULT_PROJECT_STORE_ROOT,
|
|
5630
|
+
...spec.runtime,
|
|
5631
|
+
...spec.evaluation ? { evaluation: spec.evaluation } : {}
|
|
5632
|
+
}, specPath);
|
|
5633
|
+
}
|
|
5634
|
+
async function resolveProjectConfig(specPath, configPath, warn, loaded) {
|
|
5635
|
+
const input = loaded ?? (existsSync3(specPath) ? await loadLocalRunInput(specPath) : void 0);
|
|
5636
|
+
const spec = input && input.kind !== "request" ? input.spec : void 0;
|
|
5637
|
+
const candidate = configPath ? resolve14(configPath) : join11(dirname10(resolve14(specPath)), "local-runner.json");
|
|
5638
|
+
if (configPath || existsSync3(candidate)) {
|
|
5639
|
+
const legacy = parseLocalRunnerConfig(JSON.parse(await readFile9(candidate, "utf8")), candidate);
|
|
5640
|
+
if (spec?.runtime || spec?.evaluation) {
|
|
5641
|
+
throw new Error("Conflicting configuration sources: tunedtensor.json contains runtime/evaluation settings and a legacy runner config is present. Migrate to tunedtensor.json; do not silently override the spec.");
|
|
5642
|
+
}
|
|
5643
|
+
warn?.("Legacy local-runner.json detected; migrate its settings into tunedtensor.json with tt pipeline migrate.");
|
|
5644
|
+
return { config: legacy, legacyConfigPath: candidate };
|
|
5645
|
+
}
|
|
5646
|
+
return { config: spec ? configForSpec(spec, specPath) : parseLocalRunnerConfig({}) };
|
|
5647
|
+
}
|
|
5648
|
+
|
|
5384
5649
|
// src/local-runtime/capability-profiles.ts
|
|
5385
5650
|
var GIB = 1024 ** 3;
|
|
5386
5651
|
function gib(n) {
|
|
@@ -5678,7 +5943,7 @@ function warningsFromSnapshot(report, target) {
|
|
|
5678
5943
|
import { constants as constants2 } from "fs";
|
|
5679
5944
|
import { access as access2, statfs } from "fs/promises";
|
|
5680
5945
|
import { cpus, freemem, homedir as homedir5, totalmem, type } from "os";
|
|
5681
|
-
import { dirname as
|
|
5946
|
+
import { dirname as dirname11, join as join12, resolve as resolve15 } from "path";
|
|
5682
5947
|
import { spawn as spawn2 } from "child_process";
|
|
5683
5948
|
async function runCommand(command, args, options = {}) {
|
|
5684
5949
|
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
@@ -5832,7 +6097,7 @@ async function queryNvidiaGpus(env) {
|
|
|
5832
6097
|
return { result: table.code === 0 || !last ? table : last, gpus };
|
|
5833
6098
|
}
|
|
5834
6099
|
async function diskCheck(name, path) {
|
|
5835
|
-
const resolvedPath =
|
|
6100
|
+
const resolvedPath = resolve15(path);
|
|
5836
6101
|
try {
|
|
5837
6102
|
await access2(resolvedPath, constants2.R_OK);
|
|
5838
6103
|
const fs = await statfs(resolvedPath);
|
|
@@ -5847,7 +6112,7 @@ async function diskCheck(name, path) {
|
|
|
5847
6112
|
} catch (error) {
|
|
5848
6113
|
let probe = resolvedPath;
|
|
5849
6114
|
for (let i = 0; i < 6; i += 1) {
|
|
5850
|
-
const parent =
|
|
6115
|
+
const parent = dirname11(probe);
|
|
5851
6116
|
if (parent === probe) break;
|
|
5852
6117
|
probe = parent;
|
|
5853
6118
|
try {
|
|
@@ -5933,10 +6198,10 @@ async function collectHostInventory(options = {}) {
|
|
|
5933
6198
|
const cwd = options.cwd ?? process.cwd();
|
|
5934
6199
|
const nodeVersion = process.versions.node;
|
|
5935
6200
|
const nodeMajor = Number(nodeVersion.split(".")[0]);
|
|
5936
|
-
const artifactRoot =
|
|
5937
|
-
const storeRoot =
|
|
5938
|
-
const modelCache =
|
|
5939
|
-
config?.paths.modelCache ?? env.HF_HOME ??
|
|
6201
|
+
const artifactRoot = resolve15(cwd, config?.artifactRoot ?? join12(cwd, DEFAULT_ARTIFACT_ROOT));
|
|
6202
|
+
const storeRoot = resolve15(config?.storeRoot ?? defaultStoreRoot(env));
|
|
6203
|
+
const modelCache = resolve15(
|
|
6204
|
+
config?.paths.modelCache ?? env.HF_HOME ?? join12(homedir5(), ".cache", "huggingface")
|
|
5940
6205
|
);
|
|
5941
6206
|
const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
|
|
5942
6207
|
diskCheck("artifact-root", artifactRoot),
|
|
@@ -5982,17 +6247,17 @@ async function collectHostInventory(options = {}) {
|
|
|
5982
6247
|
}
|
|
5983
6248
|
|
|
5984
6249
|
// src/local-runtime/hardware-snapshot.ts
|
|
5985
|
-
import { chmod as chmod2, mkdir as mkdir10, readFile as
|
|
5986
|
-
import { dirname as
|
|
6250
|
+
import { chmod as chmod2, mkdir as mkdir10, readFile as readFile10, writeFile as writeFile7 } from "fs/promises";
|
|
6251
|
+
import { dirname as dirname12, join as join13 } from "path";
|
|
5987
6252
|
var HARDWARE_SNAPSHOT_VERSION = 1;
|
|
5988
6253
|
var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
5989
6254
|
function hardwareSnapshotPath(env = process.env) {
|
|
5990
|
-
return
|
|
6255
|
+
return join13(getTunedTensorHome(env), "hardware.json");
|
|
5991
6256
|
}
|
|
5992
6257
|
async function readHardwareSnapshot(env = process.env) {
|
|
5993
6258
|
const path = hardwareSnapshotPath(env);
|
|
5994
6259
|
try {
|
|
5995
|
-
const parsed = JSON.parse(await
|
|
6260
|
+
const parsed = JSON.parse(await readFile10(path, "utf8"));
|
|
5996
6261
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return void 0;
|
|
5997
6262
|
const snapshot = parsed;
|
|
5998
6263
|
if (snapshot.version !== HARDWARE_SNAPSHOT_VERSION) return void 0;
|
|
@@ -6009,7 +6274,7 @@ async function readHardwareSnapshot(env = process.env) {
|
|
|
6009
6274
|
}
|
|
6010
6275
|
async function writeHardwareSnapshot(snapshot, env = process.env) {
|
|
6011
6276
|
const path = hardwareSnapshotPath(env);
|
|
6012
|
-
await mkdir10(
|
|
6277
|
+
await mkdir10(dirname12(path), { recursive: true, mode: 448 });
|
|
6013
6278
|
const gpu = snapshot.capabilities.gpu;
|
|
6014
6279
|
const record = {
|
|
6015
6280
|
version: HARDWARE_SNAPSHOT_VERSION,
|
|
@@ -6088,6 +6353,7 @@ export {
|
|
|
6088
6353
|
resolveModelLoader,
|
|
6089
6354
|
assertUsableModelArtifact,
|
|
6090
6355
|
DEFAULT_ARTIFACT_ROOT,
|
|
6356
|
+
DEFAULT_PROJECT_STORE_ROOT,
|
|
6091
6357
|
DEFAULT_FOUNDATION_RUNS_DIR,
|
|
6092
6358
|
expandUserPath,
|
|
6093
6359
|
getConfigDir,
|
|
@@ -6100,12 +6366,9 @@ export {
|
|
|
6100
6366
|
behaviorSpecSchema,
|
|
6101
6367
|
specSnapshotSchema,
|
|
6102
6368
|
foundationHyperparametersSchema,
|
|
6103
|
-
localFoundationSpecFileSchema,
|
|
6104
6369
|
datasetPrebuiltSchema,
|
|
6105
6370
|
fineTuneHyperparametersSchema,
|
|
6106
6371
|
fineTuneRunRequestSchema,
|
|
6107
|
-
localAdapterSpecFileSchema,
|
|
6108
|
-
localBehaviorSpecFileSchema,
|
|
6109
6372
|
evalExampleResultSchema,
|
|
6110
6373
|
jsonFieldMetricsSchema,
|
|
6111
6374
|
evalSplitSchema,
|
|
@@ -6114,6 +6377,10 @@ export {
|
|
|
6114
6377
|
trainingReportSchema,
|
|
6115
6378
|
runReportSchema,
|
|
6116
6379
|
localRunnerConfigSchema,
|
|
6380
|
+
projectRuntimeSchema,
|
|
6381
|
+
localFoundationSpecFileSchema,
|
|
6382
|
+
localAdapterSpecFileSchema,
|
|
6383
|
+
localBehaviorSpecFileSchema,
|
|
6117
6384
|
isFoundationSpecFile,
|
|
6118
6385
|
DEFAULT_LOCAL_SPEC_PATH,
|
|
6119
6386
|
localPathsOverlap,
|
|
@@ -6129,6 +6396,16 @@ export {
|
|
|
6129
6396
|
loadLocalRunInput,
|
|
6130
6397
|
specHash,
|
|
6131
6398
|
validateBehaviorSpec,
|
|
6399
|
+
pipelineFromFoundationHyperparameters,
|
|
6400
|
+
validatePipeline,
|
|
6401
|
+
canonicalPipeline,
|
|
6402
|
+
pipelineForRunInput,
|
|
6403
|
+
createExecutionPlan,
|
|
6404
|
+
assertFoundationSettingsMatch,
|
|
6405
|
+
canonicalFoundationPipeline2 as canonicalFoundationPipeline,
|
|
6406
|
+
canonicalJson2 as canonicalJson,
|
|
6407
|
+
isFoundationPipeline3 as isFoundationPipeline,
|
|
6408
|
+
parsePipeline3 as parsePipeline,
|
|
6132
6409
|
normalizeChatJsonlForRelocation,
|
|
6133
6410
|
buildSystemMessage,
|
|
6134
6411
|
exampleToChatRow,
|
|
@@ -6172,10 +6449,12 @@ export {
|
|
|
6172
6449
|
fingerprintLocalBaseModel,
|
|
6173
6450
|
runLocalPipeline,
|
|
6174
6451
|
runLocalFineTune,
|
|
6452
|
+
configForSpec,
|
|
6453
|
+
resolveProjectConfig,
|
|
6175
6454
|
warningsFromSnapshot,
|
|
6176
6455
|
readHardwareSnapshot,
|
|
6177
6456
|
formatHostStatusLine,
|
|
6178
6457
|
formatAgentHostBlock,
|
|
6179
6458
|
assessHardware
|
|
6180
6459
|
};
|
|
6181
|
-
//# sourceMappingURL=chunk-
|
|
6460
|
+
//# sourceMappingURL=chunk-U5HNS367.js.map
|