@tuned-tensor/cli 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -28
- package/dist/{chunk-KRKISPHG.js → chunk-4Q54NHKR.js} +435 -40
- package/dist/chunk-4Q54NHKR.js.map +1 -0
- package/dist/index.js +355 -552
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +26 -39
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/aws-gpu.md +28 -27
- package/docs/local-runtime/foundation-long-runs.md +2 -2
- package/docs/local-runtime/serving.md +3 -3
- package/docs/local-runtime/spark.md +12 -17
- package/docs/spec-architecture.html +19 -19
- package/docs/spec-workflow.md +33 -8
- package/examples/single-spec/adapter/tunedtensor.json +49 -0
- package/examples/single-spec/foundation/tunedtensor.json +31 -0
- package/package.json +4 -3
- package/training/adapter/src/serve.py +6 -0
- package/dist/chunk-KRKISPHG.js.map +0 -1
|
@@ -836,6 +836,7 @@ async function assertUsableModelArtifact(uri) {
|
|
|
836
836
|
|
|
837
837
|
// src/local-runtime/contracts.ts
|
|
838
838
|
import { z } from "zod";
|
|
839
|
+
import { pipelineDocumentSchema, isFoundationPipeline, parsePipeline } from "@tuned-tensor/pipeline-contract";
|
|
839
840
|
|
|
840
841
|
// src/paths.ts
|
|
841
842
|
import { existsSync } from "fs";
|
|
@@ -946,12 +947,6 @@ var foundationHyperparametersSchema = z.object({
|
|
|
946
947
|
keep_checkpoints: z.number().int().min(1).max(100).optional(),
|
|
947
948
|
log_interval_steps: z.number().int().min(1).max(1e6).optional()
|
|
948
949
|
}).strict();
|
|
949
|
-
var localFoundationSpecFileSchema = behaviorSpecSchema.extend({
|
|
950
|
-
engine: z.literal("foundation"),
|
|
951
|
-
id: z.string().uuid().optional(),
|
|
952
|
-
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
953
|
-
foundation: foundationHyperparametersSchema
|
|
954
|
-
});
|
|
955
950
|
var datasetPrebuiltSchema = z.object({
|
|
956
951
|
training: z.string().min(1),
|
|
957
952
|
validation: z.string().min(1).optional(),
|
|
@@ -996,16 +991,6 @@ var fineTuneRunRequestSchema = z.object({
|
|
|
996
991
|
});
|
|
997
992
|
}
|
|
998
993
|
});
|
|
999
|
-
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
1000
|
-
engine: z.literal("adapter").optional(),
|
|
1001
|
-
id: z.string().uuid().optional(),
|
|
1002
|
-
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
1003
|
-
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
1004
|
-
}).strict();
|
|
1005
|
-
var localBehaviorSpecFileSchema = z.union([
|
|
1006
|
-
localFoundationSpecFileSchema,
|
|
1007
|
-
localAdapterSpecFileSchema
|
|
1008
|
-
]);
|
|
1009
994
|
var evalExampleResultSchema = z.object({
|
|
1010
995
|
prompt: z.string(),
|
|
1011
996
|
expected: z.string(),
|
|
@@ -1196,6 +1181,42 @@ var localRunnerConfigSchema = z.object({
|
|
|
1196
1181
|
baselineCache: true
|
|
1197
1182
|
})
|
|
1198
1183
|
}).strict();
|
|
1184
|
+
var projectRuntimeSchema = z.object({
|
|
1185
|
+
gpu: localRunnerConfigSchema.shape.gpu,
|
|
1186
|
+
artifactRoot: z.string().min(1).optional(),
|
|
1187
|
+
storeRoot: z.string().min(1).optional(),
|
|
1188
|
+
paths: z.object({ baseModel: z.string().min(1).optional(), modelCache: z.string().min(1).optional() }).strict().optional()
|
|
1189
|
+
}).strict();
|
|
1190
|
+
var projectWorkflowFields = {
|
|
1191
|
+
evaluation: evaluationConfigSchema.optional(),
|
|
1192
|
+
runtime: projectRuntimeSchema.optional(),
|
|
1193
|
+
pipeline: pipelineDocumentSchema.optional()
|
|
1194
|
+
};
|
|
1195
|
+
var localFoundationSpecFileSchema = behaviorSpecSchema.extend({
|
|
1196
|
+
...projectWorkflowFields,
|
|
1197
|
+
engine: z.literal("foundation"),
|
|
1198
|
+
id: z.string().uuid().optional(),
|
|
1199
|
+
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
1200
|
+
foundation: foundationHyperparametersSchema
|
|
1201
|
+
});
|
|
1202
|
+
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
1203
|
+
...projectWorkflowFields,
|
|
1204
|
+
engine: z.literal("adapter").optional(),
|
|
1205
|
+
id: z.string().uuid().optional(),
|
|
1206
|
+
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
1207
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
1208
|
+
}).strict();
|
|
1209
|
+
var localBehaviorSpecFileSchema = z.union([
|
|
1210
|
+
localFoundationSpecFileSchema,
|
|
1211
|
+
localAdapterSpecFileSchema
|
|
1212
|
+
]).superRefine((spec, ctx) => {
|
|
1213
|
+
if (spec.pipeline && isFoundationPipeline(parsePipeline(spec.pipeline)) !== (spec.engine === "foundation")) {
|
|
1214
|
+
ctx.addIssue({ code: "custom", path: ["pipeline"], message: "Pipeline engine must match the spec engine." });
|
|
1215
|
+
}
|
|
1216
|
+
if (spec.engine === "foundation" && (spec.evaluation || spec.runtime?.storeRoot || spec.runtime?.paths)) {
|
|
1217
|
+
ctx.addIssue({ code: "custom", path: ["runtime"], message: "Foundation uses foundation settings for evaluation and checkpoints; runtime.artifactRoot and runtime.gpu are supported." });
|
|
1218
|
+
}
|
|
1219
|
+
});
|
|
1199
1220
|
function isFoundationSpecFile(spec) {
|
|
1200
1221
|
return spec.engine === "foundation";
|
|
1201
1222
|
}
|
|
@@ -1206,6 +1227,116 @@ import { readFile as readFile2, stat as stat2, writeFile as writeFile2 } from "f
|
|
|
1206
1227
|
import { dirname as dirname2, isAbsolute as isAbsolute3, relative as relative2, resolve as resolve4, sep as sep2 } from "path";
|
|
1207
1228
|
import { homedir as homedir2 } from "os";
|
|
1208
1229
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
1230
|
+
|
|
1231
|
+
// src/eval/rules.ts
|
|
1232
|
+
function validateEvalCases(spec) {
|
|
1233
|
+
const errors = [];
|
|
1234
|
+
const cases = spec.eval_cases ?? [];
|
|
1235
|
+
for (const [caseIndex, evalCase] of cases.entries()) {
|
|
1236
|
+
const label = `eval_cases[${caseIndex}]`;
|
|
1237
|
+
if (!evalCase || typeof evalCase !== "object") {
|
|
1238
|
+
errors.push(`${label} must be an object`);
|
|
1239
|
+
continue;
|
|
1240
|
+
}
|
|
1241
|
+
if (typeof evalCase.input !== "string" || evalCase.input.trim().length === 0) {
|
|
1242
|
+
errors.push(`${label}.input must be a non-empty string`);
|
|
1243
|
+
}
|
|
1244
|
+
if (evalCase.runtime !== "python") {
|
|
1245
|
+
errors.push(`${label}.runtime must be "python"`);
|
|
1246
|
+
}
|
|
1247
|
+
if (!Array.isArray(evalCase.tests) || evalCase.tests.length === 0) {
|
|
1248
|
+
errors.push(`${label}.tests must contain at least one test`);
|
|
1249
|
+
continue;
|
|
1250
|
+
}
|
|
1251
|
+
for (const [testIndex, test] of evalCase.tests.entries()) {
|
|
1252
|
+
const testLabel = `${label}.tests[${testIndex}]`;
|
|
1253
|
+
if (!test || typeof test !== "object") {
|
|
1254
|
+
errors.push(`${testLabel} must be an object`);
|
|
1255
|
+
continue;
|
|
1256
|
+
}
|
|
1257
|
+
if (test.name !== void 0 && (typeof test.name !== "string" || test.name.trim().length === 0)) {
|
|
1258
|
+
errors.push(`${testLabel}.name must be a non-empty string when provided`);
|
|
1259
|
+
}
|
|
1260
|
+
if (test.args !== void 0 && (!Array.isArray(test.args) || !test.args.every((arg) => typeof arg === "string"))) {
|
|
1261
|
+
errors.push(`${testLabel}.args must be an array of strings`);
|
|
1262
|
+
}
|
|
1263
|
+
if (test.stdin !== void 0 && typeof test.stdin !== "string") {
|
|
1264
|
+
errors.push(`${testLabel}.stdin must be a string`);
|
|
1265
|
+
}
|
|
1266
|
+
if (test.expected_exit_code !== void 0 && (typeof test.expected_exit_code !== "number" || !Number.isInteger(test.expected_exit_code) || test.expected_exit_code < 0 || test.expected_exit_code > 255)) {
|
|
1267
|
+
errors.push(`${testLabel}.expected_exit_code must be an integer between 0 and 255`);
|
|
1268
|
+
}
|
|
1269
|
+
if (test.expected_stdout !== void 0 && typeof test.expected_stdout !== "string") {
|
|
1270
|
+
errors.push(`${testLabel}.expected_stdout must be a string`);
|
|
1271
|
+
}
|
|
1272
|
+
validateFiles(test.files, `${testLabel}.files`, errors);
|
|
1273
|
+
validateFiles(test.expected_files, `${testLabel}.expected_files`, errors);
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
return errors;
|
|
1277
|
+
}
|
|
1278
|
+
function validateFiles(files, label, errors) {
|
|
1279
|
+
if (files === void 0) return;
|
|
1280
|
+
if (!Array.isArray(files)) {
|
|
1281
|
+
errors.push(`${label} must be an array`);
|
|
1282
|
+
return;
|
|
1283
|
+
}
|
|
1284
|
+
for (const [index, file] of files.entries()) {
|
|
1285
|
+
const fileLabel = `${label}[${index}]`;
|
|
1286
|
+
if (!file || typeof file !== "object") {
|
|
1287
|
+
errors.push(`${fileLabel} must be an object`);
|
|
1288
|
+
continue;
|
|
1289
|
+
}
|
|
1290
|
+
const row = file;
|
|
1291
|
+
if (typeof row.path !== "string" || !isSafeRelativePath(row.path)) {
|
|
1292
|
+
errors.push(`${fileLabel}.path must be a safe relative path`);
|
|
1293
|
+
}
|
|
1294
|
+
if (row.content !== void 0 && typeof row.content !== "string") {
|
|
1295
|
+
errors.push(`${fileLabel}.content must be a string`);
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
function isSafeRelativePath(value) {
|
|
1300
|
+
if (value.length === 0 || value.length > 240) return false;
|
|
1301
|
+
if (value.includes("\0") || value.startsWith("/")) return false;
|
|
1302
|
+
const normalized = value.replace(/\\/g, "/");
|
|
1303
|
+
return !normalized.split("/").some((part) => part === "" || part === "." || part === "..");
|
|
1304
|
+
}
|
|
1305
|
+
function evaluateConstraint(text, constraint) {
|
|
1306
|
+
const lower = constraint.toLowerCase();
|
|
1307
|
+
const neverMatch = lower.match(/^never (?:share|mention|include|reveal|use|say|output|return|give|provide|show|disclose|expose)\s+(.+)/);
|
|
1308
|
+
if (neverMatch) {
|
|
1309
|
+
const forbidden = neverMatch[1].replace(/['"]/g, "").trim();
|
|
1310
|
+
if (text.toLowerCase().includes(forbidden.toLowerCase())) {
|
|
1311
|
+
return { passed: false, message: `Output contains "${forbidden}"` };
|
|
1312
|
+
}
|
|
1313
|
+
return { passed: true };
|
|
1314
|
+
}
|
|
1315
|
+
const alwaysMatch = lower.match(/^always (?:include|use|start with|end with|contain|respond in|reply in)\s+(.+)/);
|
|
1316
|
+
if (alwaysMatch) {
|
|
1317
|
+
const required = alwaysMatch[1].replace(/['"]/g, "").trim();
|
|
1318
|
+
if (!text.toLowerCase().includes(required.toLowerCase())) {
|
|
1319
|
+
return { passed: false, message: `Output missing "${required}"` };
|
|
1320
|
+
}
|
|
1321
|
+
return { passed: true };
|
|
1322
|
+
}
|
|
1323
|
+
return {
|
|
1324
|
+
passed: true,
|
|
1325
|
+
message: `Constraint not enforceable by rules: "${constraint}" (will be checked by LLM judge if a provider is configured)`
|
|
1326
|
+
};
|
|
1327
|
+
}
|
|
1328
|
+
function checkConstraints(text, constraints) {
|
|
1329
|
+
return constraints.map((constraint) => {
|
|
1330
|
+
const result = evaluateConstraint(text, constraint);
|
|
1331
|
+
return {
|
|
1332
|
+
assertion: `constraint: ${constraint}`,
|
|
1333
|
+
passed: result.passed,
|
|
1334
|
+
message: result.message
|
|
1335
|
+
};
|
|
1336
|
+
});
|
|
1337
|
+
}
|
|
1338
|
+
|
|
1339
|
+
// src/local-runtime/local-project.ts
|
|
1209
1340
|
var DEFAULT_LOCAL_SPEC_PATH = "tunedtensor.json";
|
|
1210
1341
|
function resolveLocalReference(value, baseDirectory) {
|
|
1211
1342
|
if (typeof value !== "string" || !value || isAbsolute3(value) || /^[a-z][a-z0-9+.-]*:/i.test(value)) {
|
|
@@ -1233,6 +1364,18 @@ function resolveLocalRunInputPaths(raw, inputPath) {
|
|
|
1233
1364
|
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
1234
1365
|
}
|
|
1235
1366
|
}
|
|
1367
|
+
const runtime = value.runtime;
|
|
1368
|
+
if (runtime && typeof runtime === "object") {
|
|
1369
|
+
for (const key of ["artifactRoot", "storeRoot"]) if (runtime[key] !== void 0) runtime[key] = resolveLocalReference(runtime[key], baseDirectory);
|
|
1370
|
+
const gpu = runtime.gpu;
|
|
1371
|
+
if (gpu?.identityFile !== void 0) gpu.identityFile = resolveLocalReference(gpu.identityFile, baseDirectory);
|
|
1372
|
+
const paths = runtime.paths;
|
|
1373
|
+
if (paths && typeof paths === "object") {
|
|
1374
|
+
for (const key of ["baseModel", "modelCache"]) if (paths[key] !== void 0) paths[key] = resolveLocalReference(paths[key], baseDirectory);
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1377
|
+
const regression = value.evaluation?.generalRegression;
|
|
1378
|
+
if (regression?.dataset !== void 0) regression.dataset = resolveLocalReference(regression.dataset, baseDirectory);
|
|
1236
1379
|
const foundation = value.foundation;
|
|
1237
1380
|
if (foundation && typeof foundation === "object" && !Array.isArray(foundation)) {
|
|
1238
1381
|
const fields = foundation;
|
|
@@ -1385,6 +1528,9 @@ function runRequestFromLocalSpec(spec, options = {}) {
|
|
|
1385
1528
|
const {
|
|
1386
1529
|
id,
|
|
1387
1530
|
engine: _engine,
|
|
1531
|
+
evaluation: _evaluation,
|
|
1532
|
+
runtime: _runtime,
|
|
1533
|
+
pipeline: _pipeline,
|
|
1388
1534
|
hyperparameters,
|
|
1389
1535
|
dataset_prebuilt: datasetPrebuilt,
|
|
1390
1536
|
...specSnapshot
|
|
@@ -1480,6 +1626,11 @@ function validateBehaviorSpec(input) {
|
|
|
1480
1626
|
}
|
|
1481
1627
|
seen.add(key);
|
|
1482
1628
|
}
|
|
1629
|
+
for (const [index, example] of spec.examples.entries()) {
|
|
1630
|
+
for (const result of checkConstraints(example.output, spec.constraints ?? [])) {
|
|
1631
|
+
if (!result.passed) warnings.push(`examples[${index}] may violate ${result.assertion} (${result.message})`);
|
|
1632
|
+
}
|
|
1633
|
+
}
|
|
1483
1634
|
if (!spec.guidelines.length) {
|
|
1484
1635
|
warnings.push("Add guidelines to make expected behavior explicit.");
|
|
1485
1636
|
}
|
|
@@ -1489,6 +1640,199 @@ function validateBehaviorSpec(input) {
|
|
|
1489
1640
|
return { valid: errors.length === 0, errors, warnings };
|
|
1490
1641
|
}
|
|
1491
1642
|
|
|
1643
|
+
// src/pipeline.ts
|
|
1644
|
+
import {
|
|
1645
|
+
canonicalJson,
|
|
1646
|
+
isFoundationPipeline as isFoundationPipeline2,
|
|
1647
|
+
canonicalPipeline as canonicalPortablePipeline,
|
|
1648
|
+
parsePipeline as parsePipeline2,
|
|
1649
|
+
pipelineDocumentSchema as pipelineDocumentSchema2
|
|
1650
|
+
} from "@tuned-tensor/pipeline-contract";
|
|
1651
|
+
import {
|
|
1652
|
+
PIPELINE_DOCUMENT_VERSION,
|
|
1653
|
+
PIPELINE_EVALUATORS,
|
|
1654
|
+
PIPELINE_MAX_STEPS,
|
|
1655
|
+
PIPELINE_MAX_TRAIN_STEPS,
|
|
1656
|
+
PIPELINE_STEP_CAPABILITIES,
|
|
1657
|
+
PIPELINE_STEP_USES,
|
|
1658
|
+
PIPELINE_TARGETS,
|
|
1659
|
+
canonicalFoundationPipeline as canonicalFoundationPipeline2,
|
|
1660
|
+
canonicalJson as canonicalJson2,
|
|
1661
|
+
isFoundationPipeline as isFoundationPipeline3,
|
|
1662
|
+
parsePipeline as parsePipeline3,
|
|
1663
|
+
pipelineDocumentSchema as pipelineDocumentSchema3,
|
|
1664
|
+
pipelineHash
|
|
1665
|
+
} from "@tuned-tensor/pipeline-contract";
|
|
1666
|
+
function stepReferences(step) {
|
|
1667
|
+
if (step.uses === "evaluate") return step.with.model === "base" ? [] : [step.with.model];
|
|
1668
|
+
if (step.uses === "compare") return [step.with.before, step.with.after];
|
|
1669
|
+
if (step.uses === "pretrain") return [step.with.tokenizer];
|
|
1670
|
+
if (step.uses === "finetune" || step.uses === "rl") return [step.with.model];
|
|
1671
|
+
return [];
|
|
1672
|
+
}
|
|
1673
|
+
function pipelineFromFoundationHyperparameters(name, hp) {
|
|
1674
|
+
const lastTrainId = hp.rl_steps > 0 ? "rl" : "sft";
|
|
1675
|
+
const steps = [
|
|
1676
|
+
{
|
|
1677
|
+
id: "tokenize",
|
|
1678
|
+
uses: "tokenize",
|
|
1679
|
+
target: "local",
|
|
1680
|
+
with: { vocabSize: hp.vocab_size, maxChars: hp.max_chars }
|
|
1681
|
+
},
|
|
1682
|
+
{
|
|
1683
|
+
id: "pretrain",
|
|
1684
|
+
uses: "pretrain",
|
|
1685
|
+
target: "local",
|
|
1686
|
+
with: {
|
|
1687
|
+
tokenizer: { from: "tokenize.tokenizer" },
|
|
1688
|
+
depth: hp.depth,
|
|
1689
|
+
steps: hp.pretrain_steps,
|
|
1690
|
+
batchSize: hp.batch_size,
|
|
1691
|
+
sequenceLength: hp.sequence_length,
|
|
1692
|
+
nprocPerNode: hp.nproc_per_node
|
|
1693
|
+
}
|
|
1694
|
+
},
|
|
1695
|
+
{
|
|
1696
|
+
id: "bpb",
|
|
1697
|
+
uses: "evaluate",
|
|
1698
|
+
target: "local",
|
|
1699
|
+
with: { model: { from: "pretrain.model" }, evaluator: "bpb" }
|
|
1700
|
+
},
|
|
1701
|
+
{
|
|
1702
|
+
id: "sft",
|
|
1703
|
+
uses: "finetune",
|
|
1704
|
+
target: "local",
|
|
1705
|
+
with: {
|
|
1706
|
+
model: { from: "pretrain.model" },
|
|
1707
|
+
steps: hp.finetune_steps,
|
|
1708
|
+
batchSize: hp.batch_size
|
|
1709
|
+
}
|
|
1710
|
+
},
|
|
1711
|
+
{
|
|
1712
|
+
id: "chat",
|
|
1713
|
+
uses: "evaluate",
|
|
1714
|
+
target: "local",
|
|
1715
|
+
with: { model: { from: "sft.model" }, evaluator: "chat" }
|
|
1716
|
+
}
|
|
1717
|
+
];
|
|
1718
|
+
if (hp.rl_steps > 0) {
|
|
1719
|
+
steps.push(
|
|
1720
|
+
{
|
|
1721
|
+
id: "rl",
|
|
1722
|
+
uses: "rl",
|
|
1723
|
+
target: "local",
|
|
1724
|
+
with: { model: { from: "sft.model" }, steps: hp.rl_steps }
|
|
1725
|
+
},
|
|
1726
|
+
{
|
|
1727
|
+
id: "chat_rl",
|
|
1728
|
+
uses: "evaluate",
|
|
1729
|
+
target: "local",
|
|
1730
|
+
with: { model: { from: "rl.model" }, evaluator: "chat" }
|
|
1731
|
+
}
|
|
1732
|
+
);
|
|
1733
|
+
}
|
|
1734
|
+
steps.push({
|
|
1735
|
+
id: "infer",
|
|
1736
|
+
uses: "evaluate",
|
|
1737
|
+
target: "local",
|
|
1738
|
+
with: { model: { from: `${lastTrainId}.model` }, evaluator: "inference" }
|
|
1739
|
+
});
|
|
1740
|
+
return {
|
|
1741
|
+
version: 1,
|
|
1742
|
+
name,
|
|
1743
|
+
runtime: { engine: "foundation" },
|
|
1744
|
+
steps
|
|
1745
|
+
};
|
|
1746
|
+
}
|
|
1747
|
+
function producerId(reference) {
|
|
1748
|
+
return reference.from.slice(0, reference.from.lastIndexOf("."));
|
|
1749
|
+
}
|
|
1750
|
+
function validatePipeline(pipeline) {
|
|
1751
|
+
const structural = pipelineDocumentSchema2.safeParse(pipeline);
|
|
1752
|
+
if (!structural.success) {
|
|
1753
|
+
return structural.error.issues.map((issue) => {
|
|
1754
|
+
const path = issue.path.map(String).join(".");
|
|
1755
|
+
return path ? `${path}: ${issue.message}` : issue.message;
|
|
1756
|
+
});
|
|
1757
|
+
}
|
|
1758
|
+
try {
|
|
1759
|
+
parsePipeline2(pipeline);
|
|
1760
|
+
return [];
|
|
1761
|
+
} catch (error) {
|
|
1762
|
+
return [error instanceof Error ? error.message : String(error)];
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
function canonicalPipeline(target = "local") {
|
|
1766
|
+
return canonicalPortablePipeline(target);
|
|
1767
|
+
}
|
|
1768
|
+
function pipelineForRunInput(input, requested) {
|
|
1769
|
+
if (input.kind === "request") {
|
|
1770
|
+
assertLocalRunInputReady(input.request);
|
|
1771
|
+
} else {
|
|
1772
|
+
const validation = validateBehaviorSpec(input);
|
|
1773
|
+
if (!validation.valid) {
|
|
1774
|
+
throw new Error(`Invalid behavior spec:
|
|
1775
|
+
${validation.errors.join("\n")}`);
|
|
1776
|
+
}
|
|
1777
|
+
}
|
|
1778
|
+
const foundation = input.kind === "foundation-spec";
|
|
1779
|
+
const embedded = input.kind === "request" ? void 0 : input.spec.pipeline;
|
|
1780
|
+
if (embedded && requested !== void 0 && canonicalJson(parsePipeline2(embedded)) !== canonicalJson(parsePipeline2(requested))) {
|
|
1781
|
+
throw new Error("Conflicting pipeline settings: update tunedtensor.json.pipeline before running a different recipe.");
|
|
1782
|
+
}
|
|
1783
|
+
const recipe = requested !== void 0 ? requested : embedded ?? (foundation ? pipelineFromFoundationHyperparameters(input.spec.name, input.spec.foundation) : canonicalPipeline("local"));
|
|
1784
|
+
const pipeline = parsePipeline2(recipe);
|
|
1785
|
+
if (isFoundationPipeline2(pipeline) !== foundation) {
|
|
1786
|
+
const recipeEngine = isFoundationPipeline2(pipeline) ? "a foundation" : "an adapter";
|
|
1787
|
+
const specEngine = foundation ? "a foundation" : "an adapter";
|
|
1788
|
+
throw new Error(`Pipeline is ${recipeEngine} recipe, but the behavior spec is ${specEngine} spec. Use a matching recipe or omit --file.`);
|
|
1789
|
+
}
|
|
1790
|
+
if (foundation) assertFoundationSettingsMatch(input.spec.foundation, pipeline);
|
|
1791
|
+
return pipeline;
|
|
1792
|
+
}
|
|
1793
|
+
function createExecutionPlan(pipeline, selection = {}) {
|
|
1794
|
+
const errors = validatePipeline(pipeline);
|
|
1795
|
+
if (errors.length) throw new Error(`Invalid pipeline:
|
|
1796
|
+
- ${errors.join("\n- ")}`);
|
|
1797
|
+
const normalized = parsePipeline2(pipeline);
|
|
1798
|
+
const allIds = new Set(normalized.steps.map((step) => step.id));
|
|
1799
|
+
for (const id of [...selection.only ?? [], ...selection.skip ?? []]) {
|
|
1800
|
+
if (!allIds.has(id)) throw new Error(`Unknown pipeline step: ${id}.`);
|
|
1801
|
+
}
|
|
1802
|
+
const only = selection.only?.length ? new Set(selection.only) : void 0;
|
|
1803
|
+
const skip = new Set(selection.skip ?? []);
|
|
1804
|
+
const steps = normalized.steps.filter((step) => (!only || only.has(step.id)) && !skip.has(step.id));
|
|
1805
|
+
if (!steps.length) throw new Error("Selection leaves no pipeline steps to run.");
|
|
1806
|
+
const retained = new Map(steps.map((step) => [step.id, step]));
|
|
1807
|
+
const resolved = steps.map((step) => {
|
|
1808
|
+
const transfers = [];
|
|
1809
|
+
for (const reference of stepReferences(step)) {
|
|
1810
|
+
const producer = retained.get(producerId(reference));
|
|
1811
|
+
if (!producer) throw new Error(`Step ${step.id} has a dependency on ${producerId(reference)}, which was omitted by --only or --skip.`);
|
|
1812
|
+
if (producer.target !== step.target) transfers.push({ from: reference.from, from_target: producer.target, to_target: step.target });
|
|
1813
|
+
}
|
|
1814
|
+
return { ...step, transfers };
|
|
1815
|
+
});
|
|
1816
|
+
return { version: 1, ...normalized.name ? { name: normalized.name } : {}, steps: resolved };
|
|
1817
|
+
}
|
|
1818
|
+
function assertFoundationSettingsMatch(hp, plan) {
|
|
1819
|
+
const settings = {
|
|
1820
|
+
tokenize: { vocabSize: hp.vocab_size, maxChars: hp.max_chars },
|
|
1821
|
+
pretrain: { depth: hp.depth, steps: hp.pretrain_steps, batchSize: hp.batch_size, sequenceLength: hp.sequence_length, nprocPerNode: hp.nproc_per_node },
|
|
1822
|
+
finetune: { steps: hp.finetune_steps, batchSize: hp.batch_size },
|
|
1823
|
+
rl: { steps: hp.rl_steps }
|
|
1824
|
+
};
|
|
1825
|
+
for (const step of plan.steps) {
|
|
1826
|
+
const fields = "with" in step ? step.with : void 0;
|
|
1827
|
+
for (const [key, expected] of Object.entries(settings[step.uses] ?? {})) {
|
|
1828
|
+
if (fields?.[key] !== void 0 && fields[key] !== expected) {
|
|
1829
|
+
throw new Error(`Pipeline step ${step.id}.${key} conflicts with the behavior spec. Edit tunedtensor.json and regenerate the recipe, or omit --file.`);
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
if (step.uses === "rl" && hp.rl_steps === 0) throw new Error("The behavior spec disables RL; update foundation.rl_steps before adding an RL stage.");
|
|
1833
|
+
}
|
|
1834
|
+
}
|
|
1835
|
+
|
|
1492
1836
|
// src/local-runtime/dataset.ts
|
|
1493
1837
|
import { readFile as readFile3 } from "fs/promises";
|
|
1494
1838
|
async function loadNormalizedChatJsonl(path) {
|
|
@@ -3294,11 +3638,11 @@ function extractJsonObject(value) {
|
|
|
3294
3638
|
}
|
|
3295
3639
|
return null;
|
|
3296
3640
|
}
|
|
3297
|
-
function
|
|
3298
|
-
if (Array.isArray(value)) return value.map(
|
|
3641
|
+
function canonicalJson3(value) {
|
|
3642
|
+
if (Array.isArray(value)) return value.map(canonicalJson3);
|
|
3299
3643
|
if (value && typeof value === "object") {
|
|
3300
3644
|
return Object.fromEntries(
|
|
3301
|
-
Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key,
|
|
3645
|
+
Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalJson3(item)])
|
|
3302
3646
|
);
|
|
3303
3647
|
}
|
|
3304
3648
|
return value;
|
|
@@ -3324,7 +3668,7 @@ function scoreJsonFields(expected, actual, configuredFields) {
|
|
|
3324
3668
|
const schemaMatch = Boolean(actualJson) && expectedKeys.length === actualKeys.length && expectedKeys.every((key, index) => key === actualKeys[index]);
|
|
3325
3669
|
const fieldResults = {};
|
|
3326
3670
|
for (const field of fields) {
|
|
3327
|
-
fieldResults[field] = Object.prototype.hasOwnProperty.call(expectedJson, field) && Boolean(actualJson) && JSON.stringify(
|
|
3671
|
+
fieldResults[field] = Object.prototype.hasOwnProperty.call(expectedJson, field) && Boolean(actualJson) && JSON.stringify(canonicalJson3(expectedJson[field])) === JSON.stringify(canonicalJson3(actualJson?.[field]));
|
|
3328
3672
|
}
|
|
3329
3673
|
const correct = Object.values(fieldResults).filter(Boolean).length;
|
|
3330
3674
|
const score = fields.length > 0 ? correct / fields.length : 0;
|
|
@@ -4224,14 +4568,14 @@ function artifactPrefix(request) {
|
|
|
4224
4568
|
runId: request.run_id
|
|
4225
4569
|
});
|
|
4226
4570
|
}
|
|
4227
|
-
function
|
|
4571
|
+
function canonicalJson4(value) {
|
|
4228
4572
|
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
|
|
4229
|
-
if (Array.isArray(value)) return `[${value.map((entry) =>
|
|
4573
|
+
if (Array.isArray(value)) return `[${value.map((entry) => canonicalJson4(entry)).join(",")}]`;
|
|
4230
4574
|
const object = value;
|
|
4231
|
-
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${
|
|
4575
|
+
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson4(object[key])}`).join(",")}}`;
|
|
4232
4576
|
}
|
|
4233
4577
|
function hashJson(value) {
|
|
4234
|
-
return createHash7("sha256").update(
|
|
4578
|
+
return createHash7("sha256").update(canonicalJson4(value)).digest("hex");
|
|
4235
4579
|
}
|
|
4236
4580
|
var packageRoot2 = localRuntimePackageRoot(import.meta.url);
|
|
4237
4581
|
async function hashFileIfPresent(path) {
|
|
@@ -5294,6 +5638,16 @@ async function runLocalPipeline(input) {
|
|
|
5294
5638
|
store,
|
|
5295
5639
|
reporter: input.reporter
|
|
5296
5640
|
});
|
|
5641
|
+
await writeJsonAtomic(join10(artifacts.runDir, "resolved-workflow.json"), {
|
|
5642
|
+
spec: input.projectSpec ?? {
|
|
5643
|
+
...input.request.spec_snapshot,
|
|
5644
|
+
hyperparameters: input.request.hyperparameters,
|
|
5645
|
+
dataset_prebuilt: input.request.dataset_prebuilt
|
|
5646
|
+
},
|
|
5647
|
+
request: input.request,
|
|
5648
|
+
runtime: input.config,
|
|
5649
|
+
pipeline
|
|
5650
|
+
});
|
|
5297
5651
|
const outputs = {};
|
|
5298
5652
|
for (const step of pipeline.steps) {
|
|
5299
5653
|
await throwIfCancelled(store, input.request);
|
|
@@ -5381,6 +5735,32 @@ async function runLocalFineTune(input) {
|
|
|
5381
5735
|
};
|
|
5382
5736
|
}
|
|
5383
5737
|
|
|
5738
|
+
// src/local-runtime/project-workflow.ts
|
|
5739
|
+
import { existsSync as existsSync3 } from "fs";
|
|
5740
|
+
import { readFile as readFile9 } from "fs/promises";
|
|
5741
|
+
import { dirname as dirname10, join as join11, resolve as resolve14 } from "path";
|
|
5742
|
+
function configForSpec(spec, specPath) {
|
|
5743
|
+
return parseLocalRunnerConfig({
|
|
5744
|
+
storeRoot: process.env.TT_LOCAL_HOME?.trim() ? expandUserPath(process.env.TT_LOCAL_HOME.trim()) : DEFAULT_PROJECT_STORE_ROOT,
|
|
5745
|
+
...spec.runtime,
|
|
5746
|
+
...spec.evaluation ? { evaluation: spec.evaluation } : {}
|
|
5747
|
+
}, specPath);
|
|
5748
|
+
}
|
|
5749
|
+
async function resolveProjectConfig(specPath, configPath, warn, loaded) {
|
|
5750
|
+
const input = loaded ?? (existsSync3(specPath) ? await loadLocalRunInput(specPath) : void 0);
|
|
5751
|
+
const spec = input && input.kind !== "request" ? input.spec : void 0;
|
|
5752
|
+
const candidate = configPath ? resolve14(configPath) : join11(dirname10(resolve14(specPath)), "local-runner.json");
|
|
5753
|
+
if (configPath || existsSync3(candidate)) {
|
|
5754
|
+
const legacy = parseLocalRunnerConfig(JSON.parse(await readFile9(candidate, "utf8")), candidate);
|
|
5755
|
+
if (spec?.runtime || spec?.evaluation) {
|
|
5756
|
+
throw new Error("Conflicting configuration sources: tunedtensor.json contains runtime/evaluation settings and a legacy runner config is present. Migrate to tunedtensor.json; do not silently override the spec.");
|
|
5757
|
+
}
|
|
5758
|
+
warn?.("Legacy local-runner.json detected; migrate its settings into tunedtensor.json with tt pipeline migrate.");
|
|
5759
|
+
return { config: legacy, legacyConfigPath: candidate };
|
|
5760
|
+
}
|
|
5761
|
+
return { config: spec ? configForSpec(spec, specPath) : parseLocalRunnerConfig({}) };
|
|
5762
|
+
}
|
|
5763
|
+
|
|
5384
5764
|
// src/local-runtime/capability-profiles.ts
|
|
5385
5765
|
var GIB = 1024 ** 3;
|
|
5386
5766
|
function gib(n) {
|
|
@@ -5678,7 +6058,7 @@ function warningsFromSnapshot(report, target) {
|
|
|
5678
6058
|
import { constants as constants2 } from "fs";
|
|
5679
6059
|
import { access as access2, statfs } from "fs/promises";
|
|
5680
6060
|
import { cpus, freemem, homedir as homedir5, totalmem, type } from "os";
|
|
5681
|
-
import { dirname as
|
|
6061
|
+
import { dirname as dirname11, join as join12, resolve as resolve15 } from "path";
|
|
5682
6062
|
import { spawn as spawn2 } from "child_process";
|
|
5683
6063
|
async function runCommand(command, args, options = {}) {
|
|
5684
6064
|
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
@@ -5832,7 +6212,7 @@ async function queryNvidiaGpus(env) {
|
|
|
5832
6212
|
return { result: table.code === 0 || !last ? table : last, gpus };
|
|
5833
6213
|
}
|
|
5834
6214
|
async function diskCheck(name, path) {
|
|
5835
|
-
const resolvedPath =
|
|
6215
|
+
const resolvedPath = resolve15(path);
|
|
5836
6216
|
try {
|
|
5837
6217
|
await access2(resolvedPath, constants2.R_OK);
|
|
5838
6218
|
const fs = await statfs(resolvedPath);
|
|
@@ -5847,7 +6227,7 @@ async function diskCheck(name, path) {
|
|
|
5847
6227
|
} catch (error) {
|
|
5848
6228
|
let probe = resolvedPath;
|
|
5849
6229
|
for (let i = 0; i < 6; i += 1) {
|
|
5850
|
-
const parent =
|
|
6230
|
+
const parent = dirname11(probe);
|
|
5851
6231
|
if (parent === probe) break;
|
|
5852
6232
|
probe = parent;
|
|
5853
6233
|
try {
|
|
@@ -5933,10 +6313,10 @@ async function collectHostInventory(options = {}) {
|
|
|
5933
6313
|
const cwd = options.cwd ?? process.cwd();
|
|
5934
6314
|
const nodeVersion = process.versions.node;
|
|
5935
6315
|
const nodeMajor = Number(nodeVersion.split(".")[0]);
|
|
5936
|
-
const artifactRoot =
|
|
5937
|
-
const storeRoot =
|
|
5938
|
-
const modelCache =
|
|
5939
|
-
config?.paths.modelCache ?? env.HF_HOME ??
|
|
6316
|
+
const artifactRoot = resolve15(cwd, config?.artifactRoot ?? join12(cwd, DEFAULT_ARTIFACT_ROOT));
|
|
6317
|
+
const storeRoot = resolve15(config?.storeRoot ?? defaultStoreRoot(env));
|
|
6318
|
+
const modelCache = resolve15(
|
|
6319
|
+
config?.paths.modelCache ?? env.HF_HOME ?? join12(homedir5(), ".cache", "huggingface")
|
|
5940
6320
|
);
|
|
5941
6321
|
const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
|
|
5942
6322
|
diskCheck("artifact-root", artifactRoot),
|
|
@@ -5982,17 +6362,17 @@ async function collectHostInventory(options = {}) {
|
|
|
5982
6362
|
}
|
|
5983
6363
|
|
|
5984
6364
|
// src/local-runtime/hardware-snapshot.ts
|
|
5985
|
-
import { chmod as chmod2, mkdir as mkdir10, readFile as
|
|
5986
|
-
import { dirname as
|
|
6365
|
+
import { chmod as chmod2, mkdir as mkdir10, readFile as readFile10, writeFile as writeFile7 } from "fs/promises";
|
|
6366
|
+
import { dirname as dirname12, join as join13 } from "path";
|
|
5987
6367
|
var HARDWARE_SNAPSHOT_VERSION = 1;
|
|
5988
6368
|
var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
|
|
5989
6369
|
function hardwareSnapshotPath(env = process.env) {
|
|
5990
|
-
return
|
|
6370
|
+
return join13(getTunedTensorHome(env), "hardware.json");
|
|
5991
6371
|
}
|
|
5992
6372
|
async function readHardwareSnapshot(env = process.env) {
|
|
5993
6373
|
const path = hardwareSnapshotPath(env);
|
|
5994
6374
|
try {
|
|
5995
|
-
const parsed = JSON.parse(await
|
|
6375
|
+
const parsed = JSON.parse(await readFile10(path, "utf8"));
|
|
5996
6376
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return void 0;
|
|
5997
6377
|
const snapshot = parsed;
|
|
5998
6378
|
if (snapshot.version !== HARDWARE_SNAPSHOT_VERSION) return void 0;
|
|
@@ -6009,7 +6389,7 @@ async function readHardwareSnapshot(env = process.env) {
|
|
|
6009
6389
|
}
|
|
6010
6390
|
async function writeHardwareSnapshot(snapshot, env = process.env) {
|
|
6011
6391
|
const path = hardwareSnapshotPath(env);
|
|
6012
|
-
await mkdir10(
|
|
6392
|
+
await mkdir10(dirname12(path), { recursive: true, mode: 448 });
|
|
6013
6393
|
const gpu = snapshot.capabilities.gpu;
|
|
6014
6394
|
const record = {
|
|
6015
6395
|
version: HARDWARE_SNAPSHOT_VERSION,
|
|
@@ -6088,6 +6468,7 @@ export {
|
|
|
6088
6468
|
resolveModelLoader,
|
|
6089
6469
|
assertUsableModelArtifact,
|
|
6090
6470
|
DEFAULT_ARTIFACT_ROOT,
|
|
6471
|
+
DEFAULT_PROJECT_STORE_ROOT,
|
|
6091
6472
|
DEFAULT_FOUNDATION_RUNS_DIR,
|
|
6092
6473
|
expandUserPath,
|
|
6093
6474
|
getConfigDir,
|
|
@@ -6100,12 +6481,9 @@ export {
|
|
|
6100
6481
|
behaviorSpecSchema,
|
|
6101
6482
|
specSnapshotSchema,
|
|
6102
6483
|
foundationHyperparametersSchema,
|
|
6103
|
-
localFoundationSpecFileSchema,
|
|
6104
6484
|
datasetPrebuiltSchema,
|
|
6105
6485
|
fineTuneHyperparametersSchema,
|
|
6106
6486
|
fineTuneRunRequestSchema,
|
|
6107
|
-
localAdapterSpecFileSchema,
|
|
6108
|
-
localBehaviorSpecFileSchema,
|
|
6109
6487
|
evalExampleResultSchema,
|
|
6110
6488
|
jsonFieldMetricsSchema,
|
|
6111
6489
|
evalSplitSchema,
|
|
@@ -6114,7 +6492,12 @@ export {
|
|
|
6114
6492
|
trainingReportSchema,
|
|
6115
6493
|
runReportSchema,
|
|
6116
6494
|
localRunnerConfigSchema,
|
|
6495
|
+
projectRuntimeSchema,
|
|
6496
|
+
localFoundationSpecFileSchema,
|
|
6497
|
+
localAdapterSpecFileSchema,
|
|
6498
|
+
localBehaviorSpecFileSchema,
|
|
6117
6499
|
isFoundationSpecFile,
|
|
6500
|
+
validateEvalCases,
|
|
6118
6501
|
DEFAULT_LOCAL_SPEC_PATH,
|
|
6119
6502
|
localPathsOverlap,
|
|
6120
6503
|
resolveLocalRunInputPaths,
|
|
@@ -6129,6 +6512,16 @@ export {
|
|
|
6129
6512
|
loadLocalRunInput,
|
|
6130
6513
|
specHash,
|
|
6131
6514
|
validateBehaviorSpec,
|
|
6515
|
+
pipelineFromFoundationHyperparameters,
|
|
6516
|
+
validatePipeline,
|
|
6517
|
+
canonicalPipeline,
|
|
6518
|
+
pipelineForRunInput,
|
|
6519
|
+
createExecutionPlan,
|
|
6520
|
+
assertFoundationSettingsMatch,
|
|
6521
|
+
canonicalFoundationPipeline2 as canonicalFoundationPipeline,
|
|
6522
|
+
canonicalJson2 as canonicalJson,
|
|
6523
|
+
isFoundationPipeline3 as isFoundationPipeline,
|
|
6524
|
+
parsePipeline3 as parsePipeline,
|
|
6132
6525
|
normalizeChatJsonlForRelocation,
|
|
6133
6526
|
buildSystemMessage,
|
|
6134
6527
|
exampleToChatRow,
|
|
@@ -6172,10 +6565,12 @@ export {
|
|
|
6172
6565
|
fingerprintLocalBaseModel,
|
|
6173
6566
|
runLocalPipeline,
|
|
6174
6567
|
runLocalFineTune,
|
|
6568
|
+
configForSpec,
|
|
6569
|
+
resolveProjectConfig,
|
|
6175
6570
|
warningsFromSnapshot,
|
|
6176
6571
|
readHardwareSnapshot,
|
|
6177
6572
|
formatHostStatusLine,
|
|
6178
6573
|
formatAgentHostBlock,
|
|
6179
6574
|
assessHardware
|
|
6180
6575
|
};
|
|
6181
|
-
//# sourceMappingURL=chunk-
|
|
6576
|
+
//# sourceMappingURL=chunk-4Q54NHKR.js.map
|