@tuned-tensor/cli 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -30
- package/dist/{chunk-6RLNBTM6.js → chunk-U5HNS367.js} +371 -40
- package/dist/chunk-U5HNS367.js.map +1 -0
- package/dist/index.js +1243 -842
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +30 -39
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/aws-gpu.md +144 -26
- package/docs/local-runtime/serving.md +2 -2
- package/docs/local-runtime/spark.md +12 -17
- package/docs/spec-architecture.html +573 -0
- package/docs/spec-workflow.md +131 -0
- package/examples/single-spec/adapter/tunedtensor.json +49 -0
- package/examples/single-spec/foundation/tunedtensor.json +31 -0
- package/package.json +6 -3
- package/training/adapter/src/serve.py +6 -0
- package/dist/chunk-6RLNBTM6.js.map +0 -1
|
@@ -836,6 +836,7 @@ async function assertUsableModelArtifact(uri) {
|
|
|
836
836
|
|
|
837
837
|
// src/local-runtime/contracts.ts
|
|
838
838
|
import { z } from "zod";
|
|
839
|
+
import { pipelineDocumentSchema, isFoundationPipeline, parsePipeline } from "@tuned-tensor/pipeline-contract";
|
|
839
840
|
|
|
840
841
|
// src/paths.ts
|
|
841
842
|
import { existsSync } from "fs";
|
|
@@ -946,12 +947,6 @@ var foundationHyperparametersSchema = z.object({
|
|
|
946
947
|
keep_checkpoints: z.number().int().min(1).max(100).optional(),
|
|
947
948
|
log_interval_steps: z.number().int().min(1).max(1e6).optional()
|
|
948
949
|
}).strict();
|
|
949
|
-
var localFoundationSpecFileSchema = behaviorSpecSchema.extend({
|
|
950
|
-
engine: z.literal("foundation"),
|
|
951
|
-
id: z.string().uuid().optional(),
|
|
952
|
-
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
953
|
-
foundation: foundationHyperparametersSchema
|
|
954
|
-
});
|
|
955
950
|
var datasetPrebuiltSchema = z.object({
|
|
956
951
|
training: z.string().min(1),
|
|
957
952
|
validation: z.string().min(1).optional(),
|
|
@@ -996,16 +991,6 @@ var fineTuneRunRequestSchema = z.object({
|
|
|
996
991
|
});
|
|
997
992
|
}
|
|
998
993
|
});
|
|
999
|
-
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
1000
|
-
engine: z.literal("adapter").optional(),
|
|
1001
|
-
id: z.string().uuid().optional(),
|
|
1002
|
-
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
1003
|
-
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
1004
|
-
}).strict();
|
|
1005
|
-
var localBehaviorSpecFileSchema = z.union([
|
|
1006
|
-
localFoundationSpecFileSchema,
|
|
1007
|
-
localAdapterSpecFileSchema
|
|
1008
|
-
]);
|
|
1009
994
|
var evalExampleResultSchema = z.object({
|
|
1010
995
|
prompt: z.string(),
|
|
1011
996
|
expected: z.string(),
|
|
@@ -1196,6 +1181,42 @@ var localRunnerConfigSchema = z.object({
|
|
|
1196
1181
|
baselineCache: true
|
|
1197
1182
|
})
|
|
1198
1183
|
}).strict();
|
|
1184
|
+
var projectRuntimeSchema = z.object({
|
|
1185
|
+
gpu: localRunnerConfigSchema.shape.gpu,
|
|
1186
|
+
artifactRoot: z.string().min(1).optional(),
|
|
1187
|
+
storeRoot: z.string().min(1).optional(),
|
|
1188
|
+
paths: z.object({ baseModel: z.string().min(1).optional(), modelCache: z.string().min(1).optional() }).strict().optional()
|
|
1189
|
+
}).strict();
|
|
1190
|
+
var projectWorkflowFields = {
|
|
1191
|
+
evaluation: evaluationConfigSchema.optional(),
|
|
1192
|
+
runtime: projectRuntimeSchema.optional(),
|
|
1193
|
+
pipeline: pipelineDocumentSchema.optional()
|
|
1194
|
+
};
|
|
1195
|
+
var localFoundationSpecFileSchema = behaviorSpecSchema.extend({
|
|
1196
|
+
...projectWorkflowFields,
|
|
1197
|
+
engine: z.literal("foundation"),
|
|
1198
|
+
id: z.string().uuid().optional(),
|
|
1199
|
+
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
1200
|
+
foundation: foundationHyperparametersSchema
|
|
1201
|
+
});
|
|
1202
|
+
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
1203
|
+
...projectWorkflowFields,
|
|
1204
|
+
engine: z.literal("adapter").optional(),
|
|
1205
|
+
id: z.string().uuid().optional(),
|
|
1206
|
+
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
1207
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
1208
|
+
}).strict();
|
|
1209
|
+
var localBehaviorSpecFileSchema = z.union([
|
|
1210
|
+
localFoundationSpecFileSchema,
|
|
1211
|
+
localAdapterSpecFileSchema
|
|
1212
|
+
]).superRefine((spec, ctx) => {
|
|
1213
|
+
if (spec.pipeline && isFoundationPipeline(parsePipeline(spec.pipeline)) !== (spec.engine === "foundation")) {
|
|
1214
|
+
ctx.addIssue({ code: "custom", path: ["pipeline"], message: "Pipeline engine must match the spec engine." });
|
|
1215
|
+
}
|
|
1216
|
+
if (spec.engine === "foundation" && (spec.evaluation || spec.runtime?.storeRoot || spec.runtime?.paths)) {
|
|
1217
|
+
ctx.addIssue({ code: "custom", path: ["runtime"], message: "Foundation uses foundation settings for evaluation and checkpoints; runtime.artifactRoot and runtime.gpu are supported." });
|
|
1218
|
+
}
|
|
1219
|
+
});
|
|
1199
1220
|
function isFoundationSpecFile(spec) {
|
|
1200
1221
|
return spec.engine === "foundation";
|
|
1201
1222
|
}
|
|
@@ -1233,6 +1254,18 @@ function resolveLocalRunInputPaths(raw, inputPath) {
|
|
|
1233
1254
|
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
1234
1255
|
}
|
|
1235
1256
|
}
|
|
1257
|
+
const runtime = value.runtime;
|
|
1258
|
+
if (runtime && typeof runtime === "object") {
|
|
1259
|
+
for (const key of ["artifactRoot", "storeRoot"]) if (runtime[key] !== void 0) runtime[key] = resolveLocalReference(runtime[key], baseDirectory);
|
|
1260
|
+
const gpu = runtime.gpu;
|
|
1261
|
+
if (gpu?.identityFile !== void 0) gpu.identityFile = resolveLocalReference(gpu.identityFile, baseDirectory);
|
|
1262
|
+
const paths = runtime.paths;
|
|
1263
|
+
if (paths && typeof paths === "object") {
|
|
1264
|
+
for (const key of ["baseModel", "modelCache"]) if (paths[key] !== void 0) paths[key] = resolveLocalReference(paths[key], baseDirectory);
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
const regression = value.evaluation?.generalRegression;
|
|
1268
|
+
if (regression?.dataset !== void 0) regression.dataset = resolveLocalReference(regression.dataset, baseDirectory);
|
|
1236
1269
|
const foundation = value.foundation;
|
|
1237
1270
|
if (foundation && typeof foundation === "object" && !Array.isArray(foundation)) {
|
|
1238
1271
|
const fields = foundation;
|
|
@@ -1385,6 +1418,9 @@ function runRequestFromLocalSpec(spec, options = {}) {
|
|
|
1385
1418
|
const {
|
|
1386
1419
|
id,
|
|
1387
1420
|
engine: _engine,
|
|
1421
|
+
evaluation: _evaluation,
|
|
1422
|
+
runtime: _runtime,
|
|
1423
|
+
pipeline: _pipeline,
|
|
1388
1424
|
hyperparameters,
|
|
1389
1425
|
dataset_prebuilt: datasetPrebuilt,
|
|
1390
1426
|
...specSnapshot
|
|
@@ -1438,6 +1474,249 @@ async function loadLocalRunInput(path, options = {}) {
|
|
|
1438
1474
|
options
|
|
1439
1475
|
);
|
|
1440
1476
|
}
|
|
1477
|
+
function specHash(source) {
|
|
1478
|
+
return createHash2("sha256").update(source).digest("hex");
|
|
1479
|
+
}
|
|
1480
|
+
function validateBehaviorSpec(input) {
|
|
1481
|
+
const errors = [];
|
|
1482
|
+
const warnings = [];
|
|
1483
|
+
if (input.kind === "request") {
|
|
1484
|
+
return { valid: false, errors: ["Use a behavior spec, not a run request."], warnings };
|
|
1485
|
+
}
|
|
1486
|
+
const { spec } = input;
|
|
1487
|
+
const placeholders = input.kind === "foundation-spec" ? foundationPlaceholderIssues(input.spec) : generatedPlaceholderIssues(input.request);
|
|
1488
|
+
if (placeholders.length) {
|
|
1489
|
+
errors.push(`Edit the generated behavior spec before training: ${placeholders.join("; ")}.`);
|
|
1490
|
+
}
|
|
1491
|
+
if (input.kind === "foundation-spec") {
|
|
1492
|
+
if (input.spec.foundation.nproc_per_node !== 1) {
|
|
1493
|
+
errors.push("Foundation nproc_per_node must be 1 for this runtime.");
|
|
1494
|
+
}
|
|
1495
|
+
const nonNumericOutput = spec.examples.some(
|
|
1496
|
+
(example) => !/-?\d+(?:\.\d+)?/.test(example.output.replaceAll(",", ""))
|
|
1497
|
+
);
|
|
1498
|
+
if (input.spec.foundation.rl_steps > 0 && nonNumericOutput) {
|
|
1499
|
+
errors.push("Foundation RL requires numeric expected outputs in every example.");
|
|
1500
|
+
}
|
|
1501
|
+
}
|
|
1502
|
+
for (const key of ["name", "system_prompt"]) {
|
|
1503
|
+
if (!spec[key].trim()) errors.push(`${key} must not be blank.`);
|
|
1504
|
+
}
|
|
1505
|
+
if (spec.examples.length < 2 && !(input.kind === "spec" && input.spec.dataset_prebuilt)) {
|
|
1506
|
+
errors.push("Add at least two examples for training and evaluation.");
|
|
1507
|
+
}
|
|
1508
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1509
|
+
for (const [index, example] of spec.examples.entries()) {
|
|
1510
|
+
if (!example.input.trim() || !example.output.trim()) {
|
|
1511
|
+
errors.push(`examples[${index}] input and output must not be blank.`);
|
|
1512
|
+
}
|
|
1513
|
+
const key = example.input.trim();
|
|
1514
|
+
if (seen.has(key)) {
|
|
1515
|
+
errors.push(`examples[${index}] repeats an input; use distinct examples to avoid conflicting labels or train/eval overlap.`);
|
|
1516
|
+
}
|
|
1517
|
+
seen.add(key);
|
|
1518
|
+
}
|
|
1519
|
+
if (!spec.guidelines.length) {
|
|
1520
|
+
warnings.push("Add guidelines to make expected behavior explicit.");
|
|
1521
|
+
}
|
|
1522
|
+
if (spec.examples.length < 10) {
|
|
1523
|
+
warnings.push("This is a small example set; passing validation does not establish model quality.");
|
|
1524
|
+
}
|
|
1525
|
+
return { valid: errors.length === 0, errors, warnings };
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1528
|
+
// src/pipeline.ts
|
|
1529
|
+
import {
|
|
1530
|
+
canonicalJson,
|
|
1531
|
+
isFoundationPipeline as isFoundationPipeline2,
|
|
1532
|
+
canonicalPipeline as canonicalPortablePipeline,
|
|
1533
|
+
parsePipeline as parsePipeline2,
|
|
1534
|
+
pipelineDocumentSchema as pipelineDocumentSchema2
|
|
1535
|
+
} from "@tuned-tensor/pipeline-contract";
|
|
1536
|
+
import {
|
|
1537
|
+
PIPELINE_DOCUMENT_VERSION,
|
|
1538
|
+
PIPELINE_EVALUATORS,
|
|
1539
|
+
PIPELINE_MAX_STEPS,
|
|
1540
|
+
PIPELINE_MAX_TRAIN_STEPS,
|
|
1541
|
+
PIPELINE_STEP_CAPABILITIES,
|
|
1542
|
+
PIPELINE_STEP_USES,
|
|
1543
|
+
PIPELINE_TARGETS,
|
|
1544
|
+
canonicalFoundationPipeline as canonicalFoundationPipeline2,
|
|
1545
|
+
canonicalJson as canonicalJson2,
|
|
1546
|
+
isFoundationPipeline as isFoundationPipeline3,
|
|
1547
|
+
parsePipeline as parsePipeline3,
|
|
1548
|
+
pipelineDocumentSchema as pipelineDocumentSchema3,
|
|
1549
|
+
pipelineHash
|
|
1550
|
+
} from "@tuned-tensor/pipeline-contract";
|
|
1551
|
+
function stepReferences(step) {
|
|
1552
|
+
if (step.uses === "evaluate") return step.with.model === "base" ? [] : [step.with.model];
|
|
1553
|
+
if (step.uses === "compare") return [step.with.before, step.with.after];
|
|
1554
|
+
if (step.uses === "pretrain") return [step.with.tokenizer];
|
|
1555
|
+
if (step.uses === "finetune" || step.uses === "rl") return [step.with.model];
|
|
1556
|
+
return [];
|
|
1557
|
+
}
|
|
1558
|
+
function pipelineFromFoundationHyperparameters(name, hp) {
|
|
1559
|
+
const lastTrainId = hp.rl_steps > 0 ? "rl" : "sft";
|
|
1560
|
+
const steps = [
|
|
1561
|
+
{
|
|
1562
|
+
id: "tokenize",
|
|
1563
|
+
uses: "tokenize",
|
|
1564
|
+
target: "local",
|
|
1565
|
+
with: { vocabSize: hp.vocab_size, maxChars: hp.max_chars }
|
|
1566
|
+
},
|
|
1567
|
+
{
|
|
1568
|
+
id: "pretrain",
|
|
1569
|
+
uses: "pretrain",
|
|
1570
|
+
target: "local",
|
|
1571
|
+
with: {
|
|
1572
|
+
tokenizer: { from: "tokenize.tokenizer" },
|
|
1573
|
+
depth: hp.depth,
|
|
1574
|
+
steps: hp.pretrain_steps,
|
|
1575
|
+
batchSize: hp.batch_size,
|
|
1576
|
+
sequenceLength: hp.sequence_length,
|
|
1577
|
+
nprocPerNode: hp.nproc_per_node
|
|
1578
|
+
}
|
|
1579
|
+
},
|
|
1580
|
+
{
|
|
1581
|
+
id: "bpb",
|
|
1582
|
+
uses: "evaluate",
|
|
1583
|
+
target: "local",
|
|
1584
|
+
with: { model: { from: "pretrain.model" }, evaluator: "bpb" }
|
|
1585
|
+
},
|
|
1586
|
+
{
|
|
1587
|
+
id: "sft",
|
|
1588
|
+
uses: "finetune",
|
|
1589
|
+
target: "local",
|
|
1590
|
+
with: {
|
|
1591
|
+
model: { from: "pretrain.model" },
|
|
1592
|
+
steps: hp.finetune_steps,
|
|
1593
|
+
batchSize: hp.batch_size
|
|
1594
|
+
}
|
|
1595
|
+
},
|
|
1596
|
+
{
|
|
1597
|
+
id: "chat",
|
|
1598
|
+
uses: "evaluate",
|
|
1599
|
+
target: "local",
|
|
1600
|
+
with: { model: { from: "sft.model" }, evaluator: "chat" }
|
|
1601
|
+
}
|
|
1602
|
+
];
|
|
1603
|
+
if (hp.rl_steps > 0) {
|
|
1604
|
+
steps.push(
|
|
1605
|
+
{
|
|
1606
|
+
id: "rl",
|
|
1607
|
+
uses: "rl",
|
|
1608
|
+
target: "local",
|
|
1609
|
+
with: { model: { from: "sft.model" }, steps: hp.rl_steps }
|
|
1610
|
+
},
|
|
1611
|
+
{
|
|
1612
|
+
id: "chat_rl",
|
|
1613
|
+
uses: "evaluate",
|
|
1614
|
+
target: "local",
|
|
1615
|
+
with: { model: { from: "rl.model" }, evaluator: "chat" }
|
|
1616
|
+
}
|
|
1617
|
+
);
|
|
1618
|
+
}
|
|
1619
|
+
steps.push({
|
|
1620
|
+
id: "infer",
|
|
1621
|
+
uses: "evaluate",
|
|
1622
|
+
target: "local",
|
|
1623
|
+
with: { model: { from: `${lastTrainId}.model` }, evaluator: "inference" }
|
|
1624
|
+
});
|
|
1625
|
+
return {
|
|
1626
|
+
version: 1,
|
|
1627
|
+
name,
|
|
1628
|
+
runtime: { engine: "foundation" },
|
|
1629
|
+
steps
|
|
1630
|
+
};
|
|
1631
|
+
}
|
|
1632
|
+
function producerId(reference) {
|
|
1633
|
+
return reference.from.slice(0, reference.from.lastIndexOf("."));
|
|
1634
|
+
}
|
|
1635
|
+
function validatePipeline(pipeline) {
|
|
1636
|
+
const structural = pipelineDocumentSchema2.safeParse(pipeline);
|
|
1637
|
+
if (!structural.success) {
|
|
1638
|
+
return structural.error.issues.map((issue) => {
|
|
1639
|
+
const path = issue.path.map(String).join(".");
|
|
1640
|
+
return path ? `${path}: ${issue.message}` : issue.message;
|
|
1641
|
+
});
|
|
1642
|
+
}
|
|
1643
|
+
try {
|
|
1644
|
+
parsePipeline2(pipeline);
|
|
1645
|
+
return [];
|
|
1646
|
+
} catch (error) {
|
|
1647
|
+
return [error instanceof Error ? error.message : String(error)];
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
function canonicalPipeline(target = "local") {
|
|
1651
|
+
return canonicalPortablePipeline(target);
|
|
1652
|
+
}
|
|
1653
|
+
function pipelineForRunInput(input, requested) {
|
|
1654
|
+
if (input.kind === "request") {
|
|
1655
|
+
assertLocalRunInputReady(input.request);
|
|
1656
|
+
} else {
|
|
1657
|
+
const validation = validateBehaviorSpec(input);
|
|
1658
|
+
if (!validation.valid) {
|
|
1659
|
+
throw new Error(`Invalid behavior spec:
|
|
1660
|
+
${validation.errors.join("\n")}`);
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
const foundation = input.kind === "foundation-spec";
|
|
1664
|
+
const embedded = input.kind === "request" ? void 0 : input.spec.pipeline;
|
|
1665
|
+
if (embedded && requested !== void 0 && canonicalJson(parsePipeline2(embedded)) !== canonicalJson(parsePipeline2(requested))) {
|
|
1666
|
+
throw new Error("Conflicting pipeline settings: update tunedtensor.json.pipeline before running a different recipe.");
|
|
1667
|
+
}
|
|
1668
|
+
const recipe = requested !== void 0 ? requested : embedded ?? (foundation ? pipelineFromFoundationHyperparameters(input.spec.name, input.spec.foundation) : canonicalPipeline("local"));
|
|
1669
|
+
const pipeline = parsePipeline2(recipe);
|
|
1670
|
+
if (isFoundationPipeline2(pipeline) !== foundation) {
|
|
1671
|
+
const recipeEngine = isFoundationPipeline2(pipeline) ? "a foundation" : "an adapter";
|
|
1672
|
+
const specEngine = foundation ? "a foundation" : "an adapter";
|
|
1673
|
+
throw new Error(`Pipeline is ${recipeEngine} recipe, but the behavior spec is ${specEngine} spec. Use a matching recipe or omit --file.`);
|
|
1674
|
+
}
|
|
1675
|
+
if (foundation) assertFoundationSettingsMatch(input.spec.foundation, pipeline);
|
|
1676
|
+
return pipeline;
|
|
1677
|
+
}
|
|
1678
|
+
function createExecutionPlan(pipeline, selection = {}) {
|
|
1679
|
+
const errors = validatePipeline(pipeline);
|
|
1680
|
+
if (errors.length) throw new Error(`Invalid pipeline:
|
|
1681
|
+
- ${errors.join("\n- ")}`);
|
|
1682
|
+
const normalized = parsePipeline2(pipeline);
|
|
1683
|
+
const allIds = new Set(normalized.steps.map((step) => step.id));
|
|
1684
|
+
for (const id of [...selection.only ?? [], ...selection.skip ?? []]) {
|
|
1685
|
+
if (!allIds.has(id)) throw new Error(`Unknown pipeline step: ${id}.`);
|
|
1686
|
+
}
|
|
1687
|
+
const only = selection.only?.length ? new Set(selection.only) : void 0;
|
|
1688
|
+
const skip = new Set(selection.skip ?? []);
|
|
1689
|
+
const steps = normalized.steps.filter((step) => (!only || only.has(step.id)) && !skip.has(step.id));
|
|
1690
|
+
if (!steps.length) throw new Error("Selection leaves no pipeline steps to run.");
|
|
1691
|
+
const retained = new Map(steps.map((step) => [step.id, step]));
|
|
1692
|
+
const resolved = steps.map((step) => {
|
|
1693
|
+
const transfers = [];
|
|
1694
|
+
for (const reference of stepReferences(step)) {
|
|
1695
|
+
const producer = retained.get(producerId(reference));
|
|
1696
|
+
if (!producer) throw new Error(`Step ${step.id} has a dependency on ${producerId(reference)}, which was omitted by --only or --skip.`);
|
|
1697
|
+
if (producer.target !== step.target) transfers.push({ from: reference.from, from_target: producer.target, to_target: step.target });
|
|
1698
|
+
}
|
|
1699
|
+
return { ...step, transfers };
|
|
1700
|
+
});
|
|
1701
|
+
return { version: 1, ...normalized.name ? { name: normalized.name } : {}, steps: resolved };
|
|
1702
|
+
}
|
|
1703
|
+
function assertFoundationSettingsMatch(hp, plan) {
|
|
1704
|
+
const settings = {
|
|
1705
|
+
tokenize: { vocabSize: hp.vocab_size, maxChars: hp.max_chars },
|
|
1706
|
+
pretrain: { depth: hp.depth, steps: hp.pretrain_steps, batchSize: hp.batch_size, sequenceLength: hp.sequence_length, nprocPerNode: hp.nproc_per_node },
|
|
1707
|
+
finetune: { steps: hp.finetune_steps, batchSize: hp.batch_size },
|
|
1708
|
+
rl: { steps: hp.rl_steps }
|
|
1709
|
+
};
|
|
1710
|
+
for (const step of plan.steps) {
|
|
1711
|
+
const fields = "with" in step ? step.with : void 0;
|
|
1712
|
+
for (const [key, expected] of Object.entries(settings[step.uses] ?? {})) {
|
|
1713
|
+
if (fields?.[key] !== void 0 && fields[key] !== expected) {
|
|
1714
|
+
throw new Error(`Pipeline step ${step.id}.${key} conflicts with the behavior spec. Edit tunedtensor.json and regenerate the recipe, or omit --file.`);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
if (step.uses === "rl" && hp.rl_steps === 0) throw new Error("The behavior spec disables RL; update foundation.rl_steps before adding an RL stage.");
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1441
1720
|
|
|
1442
1721
|
// src/local-runtime/dataset.ts
|
|
1443
1722
|
import { readFile as readFile3 } from "fs/promises";
|
|
@@ -3244,11 +3523,11 @@ function extractJsonObject(value) {
|
|
|
3244
3523
|
}
|
|
3245
3524
|
return null;
|
|
3246
3525
|
}
|
|
3247
|
-
function
|
|
3248
|
-
if (Array.isArray(value)) return value.map(
|
|
3526
|
+
function canonicalJson3(value) {
|
|
3527
|
+
if (Array.isArray(value)) return value.map(canonicalJson3);
|
|
3249
3528
|
if (value && typeof value === "object") {
|
|
3250
3529
|
return Object.fromEntries(
|
|
3251
|
-
Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key,
|
|
3530
|
+
Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalJson3(item)])
|
|
3252
3531
|
);
|
|
3253
3532
|
}
|
|
3254
3533
|
return value;
|
|
@@ -3274,7 +3553,7 @@ function scoreJsonFields(expected, actual, configuredFields) {
|
|
|
3274
3553
|
const schemaMatch = Boolean(actualJson) && expectedKeys.length === actualKeys.length && expectedKeys.every((key, index) => key === actualKeys[index]);
|
|
3275
3554
|
const fieldResults = {};
|
|
3276
3555
|
for (const field of fields) {
|
|
3277
|
-
fieldResults[field] = Object.prototype.hasOwnProperty.call(expectedJson, field) && Boolean(actualJson) && JSON.stringify(
|
|
3556
|
+
fieldResults[field] = Object.prototype.hasOwnProperty.call(expectedJson, field) && Boolean(actualJson) && JSON.stringify(canonicalJson3(expectedJson[field])) === JSON.stringify(canonicalJson3(actualJson?.[field]));
|
|
3278
3557
|
}
|
|
3279
3558
|
const correct = Object.values(fieldResults).filter(Boolean).length;
|
|
3280
3559
|
const score = fields.length > 0 ? correct / fields.length : 0;
|
|
@@ -4174,14 +4453,14 @@ function artifactPrefix(request) {
|
|
|
4174
4453
|
runId: request.run_id
|
|
4175
4454
|
});
|
|
4176
4455
|
}
|
|
4177
|
-
function
|
|
4456
|
+
function canonicalJson4(value) {
|
|
4178
4457
|
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
|
|
4179
|
-
if (Array.isArray(value)) return `[${value.map((entry) =>
|
|
4458
|
+
if (Array.isArray(value)) return `[${value.map((entry) => canonicalJson4(entry)).join(",")}]`;
|
|
4180
4459
|
const object = value;
|
|
4181
|
-
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${
|
|
4460
|
+
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson4(object[key])}`).join(",")}}`;
|
|
4182
4461
|
}
|
|
4183
4462
|
function hashJson(value) {
|
|
4184
|
-
return createHash7("sha256").update(
|
|
4463
|
+
return createHash7("sha256").update(canonicalJson4(value)).digest("hex");
|
|
4185
4464
|
}
|
|
4186
4465
|
var packageRoot2 = localRuntimePackageRoot(import.meta.url);
|
|
4187
4466
|
async function hashFileIfPresent(path) {
|
|
@@ -5244,6 +5523,16 @@ async function runLocalPipeline(input) {
|
|
|
5244
5523
|
store,
|
|
5245
5524
|
reporter: input.reporter
|
|
5246
5525
|
});
|
|
5526
|
+
await writeJsonAtomic(join10(artifacts.runDir, "resolved-workflow.json"), {
|
|
5527
|
+
spec: input.projectSpec ?? {
|
|
5528
|
+
...input.request.spec_snapshot,
|
|
5529
|
+
hyperparameters: input.request.hyperparameters,
|
|
5530
|
+
dataset_prebuilt: input.request.dataset_prebuilt
|
|
5531
|
+
},
|
|
5532
|
+
request: input.request,
|
|
5533
|
+
runtime: input.config,
|
|
5534
|
+
pipeline
|
|
5535
|
+
});
|
|
5247
5536
|
const outputs = {};
|
|
5248
5537
|
for (const step of pipeline.steps) {
|
|
5249
5538
|
await throwIfCancelled(store, input.request);
|
|
@@ -5331,6 +5620,32 @@ async function runLocalFineTune(input) {
|
|
|
5331
5620
|
};
|
|
5332
5621
|
}
|
|
5333
5622
|
|
|
5623
|
+
// src/local-runtime/project-workflow.ts
|
|
5624
|
+
import { existsSync as existsSync3 } from "fs";
|
|
5625
|
+
import { readFile as readFile9 } from "fs/promises";
|
|
5626
|
+
import { dirname as dirname10, join as join11, resolve as resolve14 } from "path";
|
|
5627
|
+
function configForSpec(spec, specPath) {
|
|
5628
|
+
return parseLocalRunnerConfig({
|
|
5629
|
+
storeRoot: process.env.TT_LOCAL_HOME?.trim() ? expandUserPath(process.env.TT_LOCAL_HOME.trim()) : DEFAULT_PROJECT_STORE_ROOT,
|
|
5630
|
+
...spec.runtime,
|
|
5631
|
+
...spec.evaluation ? { evaluation: spec.evaluation } : {}
|
|
5632
|
+
}, specPath);
|
|
5633
|
+
}
|
|
5634
|
+
async function resolveProjectConfig(specPath, configPath, warn, loaded) {
|
|
5635
|
+
const input = loaded ?? (existsSync3(specPath) ? await loadLocalRunInput(specPath) : void 0);
|
|
5636
|
+
const spec = input && input.kind !== "request" ? input.spec : void 0;
|
|
5637
|
+
const candidate = configPath ? resolve14(configPath) : join11(dirname10(resolve14(specPath)), "local-runner.json");
|
|
5638
|
+
if (configPath || existsSync3(candidate)) {
|
|
5639
|
+
const legacy = parseLocalRunnerConfig(JSON.parse(await readFile9(candidate, "utf8")), candidate);
|
|
5640
|
+
if (spec?.runtime || spec?.evaluation) {
|
|
5641
|
+
throw new Error("Conflicting configuration sources: tunedtensor.json contains runtime/evaluation settings and a legacy runner config is present. Migrate to tunedtensor.json; do not silently override the spec.");
|
|
5642
|
+
}
|
|
5643
|
+
warn?.("Legacy local-runner.json detected; migrate its settings into tunedtensor.json with tt pipeline migrate.");
|
|
5644
|
+
return { config: legacy, legacyConfigPath: candidate };
|
|
5645
|
+
}
|
|
5646
|
+
return { config: spec ? configForSpec(spec, specPath) : parseLocalRunnerConfig({}) };
|
|
5647
|
+
}
|
|
5648
|
+
|
|
5334
5649
|
// src/local-runtime/capability-profiles.ts
|
|
5335
5650
|
var GIB = 1024 ** 3;
|
|
5336
5651
|
function gib(n) {
|
|
@@ -5628,7 +5943,7 @@ function warningsFromSnapshot(report, target) {
|
|
|
5628
5943
|
import { constants as constants2 } from "fs";
|
|
5629
5944
|
import { access as access2, statfs } from "fs/promises";
|
|
5630
5945
|
import { cpus, freemem, homedir as homedir5, totalmem, type } from "os";
|
|
5631
|
-
import { dirname as
|
|
5946
|
+
import { dirname as dirname11, join as join12, resolve as resolve15 } from "path";
|
|
5632
5947
|
import { spawn as spawn2 } from "child_process";
|
|
5633
5948
|
async function runCommand(command, args, options = {}) {
|
|
5634
5949
|
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
@@ -5782,7 +6097,7 @@ async function queryNvidiaGpus(env) {
|
|
|
5782
6097
|
return { result: table.code === 0 || !last ? table : last, gpus };
|
|
5783
6098
|
}
|
|
5784
6099
|
async function diskCheck(name, path) {
|
|
5785
|
-
const resolvedPath =
|
|
6100
|
+
const resolvedPath = resolve15(path);
|
|
5786
6101
|
try {
|
|
5787
6102
|
await access2(resolvedPath, constants2.R_OK);
|
|
5788
6103
|
const fs = await statfs(resolvedPath);
|
|
@@ -5797,7 +6112,7 @@ async function diskCheck(name, path) {
|
|
|
5797
6112
|
} catch (error) {
|
|
5798
6113
|
let probe = resolvedPath;
|
|
5799
6114
|
for (let i = 0; i < 6; i += 1) {
|
|
5800
|
-
const parent =
|
|
6115
|
+
const parent = dirname11(probe);
|
|
5801
6116
|
if (parent === probe) break;
|
|
5802
6117
|
probe = parent;
|
|
5803
6118
|
try {
|
|
@@ -5883,10 +6198,10 @@ async function collectHostInventory(options = {}) {
|
|
|
5883
6198
|
const cwd = options.cwd ?? process.cwd();
|
|
5884
6199
|
const nodeVersion = process.versions.node;
|
|
5885
6200
|
const nodeMajor = Number(nodeVersion.split(".")[0]);
|
|
5886
|
-
const artifactRoot =
|
|
5887
|
-
const storeRoot =
|
|
5888
|
-
const modelCache =
|
|
5889
|
-
config?.paths.modelCache ?? env.HF_HOME ??
|
|
6201
|
+
const artifactRoot = resolve15(cwd, config?.artifactRoot ?? join12(cwd, DEFAULT_ARTIFACT_ROOT));
|
|
6202
|
+
const storeRoot = resolve15(config?.storeRoot ?? defaultStoreRoot(env));
|
|
6203
|
+
const modelCache = resolve15(
|
|
6204
|
+
config?.paths.modelCache ?? env.HF_HOME ?? join12(homedir5(), ".cache", "huggingface")
|
|
5890
6205
|
);
|
|
5891
6206
|
const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
|
|
5892
6207
|
diskCheck("artifact-root", artifactRoot),
|
|
@@ -5932,17 +6247,17 @@ async function collectHostInventory(options = {}) {
|
|
|
5932
6247
|
}
|
|
5933
6248
|
|
|
5934
6249
|
// src/local-runtime/hardware-snapshot.ts
|
|
5935
|
-
import { chmod as chmod2, mkdir as mkdir10, readFile as
|
|
5936
|
-
import { dirname as
|
|
6250
|
+
import { chmod as chmod2, mkdir as mkdir10, readFile as readFile10, writeFile as writeFile7 } from "fs/promises";
|
|
6251
|
+
import { dirname as dirname12, join as join13 } from "path";
|
|
5937
6252
|
var HARDWARE_SNAPSHOT_VERSION = 1;
|
|
5938
6253
|
var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
5939
6254
|
function hardwareSnapshotPath(env = process.env) {
|
|
5940
|
-
return
|
|
6255
|
+
return join13(getTunedTensorHome(env), "hardware.json");
|
|
5941
6256
|
}
|
|
5942
6257
|
async function readHardwareSnapshot(env = process.env) {
|
|
5943
6258
|
const path = hardwareSnapshotPath(env);
|
|
5944
6259
|
try {
|
|
5945
|
-
const parsed = JSON.parse(await
|
|
6260
|
+
const parsed = JSON.parse(await readFile10(path, "utf8"));
|
|
5946
6261
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return void 0;
|
|
5947
6262
|
const snapshot = parsed;
|
|
5948
6263
|
if (snapshot.version !== HARDWARE_SNAPSHOT_VERSION) return void 0;
|
|
@@ -5959,7 +6274,7 @@ async function readHardwareSnapshot(env = process.env) {
|
|
|
5959
6274
|
}
|
|
5960
6275
|
async function writeHardwareSnapshot(snapshot, env = process.env) {
|
|
5961
6276
|
const path = hardwareSnapshotPath(env);
|
|
5962
|
-
await mkdir10(
|
|
6277
|
+
await mkdir10(dirname12(path), { recursive: true, mode: 448 });
|
|
5963
6278
|
const gpu = snapshot.capabilities.gpu;
|
|
5964
6279
|
const record = {
|
|
5965
6280
|
version: HARDWARE_SNAPSHOT_VERSION,
|
|
@@ -6038,6 +6353,7 @@ export {
|
|
|
6038
6353
|
resolveModelLoader,
|
|
6039
6354
|
assertUsableModelArtifact,
|
|
6040
6355
|
DEFAULT_ARTIFACT_ROOT,
|
|
6356
|
+
DEFAULT_PROJECT_STORE_ROOT,
|
|
6041
6357
|
DEFAULT_FOUNDATION_RUNS_DIR,
|
|
6042
6358
|
expandUserPath,
|
|
6043
6359
|
getConfigDir,
|
|
@@ -6050,12 +6366,9 @@ export {
|
|
|
6050
6366
|
behaviorSpecSchema,
|
|
6051
6367
|
specSnapshotSchema,
|
|
6052
6368
|
foundationHyperparametersSchema,
|
|
6053
|
-
localFoundationSpecFileSchema,
|
|
6054
6369
|
datasetPrebuiltSchema,
|
|
6055
6370
|
fineTuneHyperparametersSchema,
|
|
6056
6371
|
fineTuneRunRequestSchema,
|
|
6057
|
-
localAdapterSpecFileSchema,
|
|
6058
|
-
localBehaviorSpecFileSchema,
|
|
6059
6372
|
evalExampleResultSchema,
|
|
6060
6373
|
jsonFieldMetricsSchema,
|
|
6061
6374
|
evalSplitSchema,
|
|
@@ -6064,6 +6377,10 @@ export {
|
|
|
6064
6377
|
trainingReportSchema,
|
|
6065
6378
|
runReportSchema,
|
|
6066
6379
|
localRunnerConfigSchema,
|
|
6380
|
+
projectRuntimeSchema,
|
|
6381
|
+
localFoundationSpecFileSchema,
|
|
6382
|
+
localAdapterSpecFileSchema,
|
|
6383
|
+
localBehaviorSpecFileSchema,
|
|
6067
6384
|
isFoundationSpecFile,
|
|
6068
6385
|
DEFAULT_LOCAL_SPEC_PATH,
|
|
6069
6386
|
localPathsOverlap,
|
|
@@ -6077,6 +6394,18 @@ export {
|
|
|
6077
6394
|
runRequestFromLocalSpec,
|
|
6078
6395
|
parseLocalRunInput,
|
|
6079
6396
|
loadLocalRunInput,
|
|
6397
|
+
specHash,
|
|
6398
|
+
validateBehaviorSpec,
|
|
6399
|
+
pipelineFromFoundationHyperparameters,
|
|
6400
|
+
validatePipeline,
|
|
6401
|
+
canonicalPipeline,
|
|
6402
|
+
pipelineForRunInput,
|
|
6403
|
+
createExecutionPlan,
|
|
6404
|
+
assertFoundationSettingsMatch,
|
|
6405
|
+
canonicalFoundationPipeline2 as canonicalFoundationPipeline,
|
|
6406
|
+
canonicalJson2 as canonicalJson,
|
|
6407
|
+
isFoundationPipeline3 as isFoundationPipeline,
|
|
6408
|
+
parsePipeline3 as parsePipeline,
|
|
6080
6409
|
normalizeChatJsonlForRelocation,
|
|
6081
6410
|
buildSystemMessage,
|
|
6082
6411
|
exampleToChatRow,
|
|
@@ -6120,10 +6449,12 @@ export {
|
|
|
6120
6449
|
fingerprintLocalBaseModel,
|
|
6121
6450
|
runLocalPipeline,
|
|
6122
6451
|
runLocalFineTune,
|
|
6452
|
+
configForSpec,
|
|
6453
|
+
resolveProjectConfig,
|
|
6123
6454
|
warningsFromSnapshot,
|
|
6124
6455
|
readHardwareSnapshot,
|
|
6125
6456
|
formatHostStatusLine,
|
|
6126
6457
|
formatAgentHostBlock,
|
|
6127
6458
|
assessHardware
|
|
6128
6459
|
};
|
|
6129
|
-
//# sourceMappingURL=chunk-
|
|
6460
|
+
//# sourceMappingURL=chunk-U5HNS367.js.map
|