@tuned-tensor/cli 0.13.0-beta.1 → 0.13.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -32
- package/dist/{chunk-2VJ3VVZF.js → chunk-YJYCGSHJ.js} +537 -536
- package/dist/chunk-YJYCGSHJ.js.map +1 -0
- package/dist/index.js +499 -554
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +39 -10
- package/dist/local-runtime.js.map +1 -1
- package/docs/local-runtime/architecture.md +6 -5
- package/docs/local-runtime/spark.md +23 -17
- package/package.json +1 -1
- package/training/local-runner/src/evaluate.py +4 -1
- package/training/local-runner/src/model_contract.py +5 -3
- package/training/local-runner/src/serve.py +1 -1
- package/dist/chunk-2VJ3VVZF.js.map +0 -1
|
@@ -558,12 +558,14 @@ async function assertArtifactManifest(manifestPath, options = {}) {
|
|
|
558
558
|
import { lstat as lstat2, readdir as readdir2 } from "fs/promises";
|
|
559
559
|
import { resolve as resolve2 } from "path";
|
|
560
560
|
import { fileURLToPath } from "url";
|
|
561
|
+
var QWEN_3_5_2B_REVISION = "15852e8c16360a2fea060d615a32b45270f8a8fc";
|
|
561
562
|
var NEMOTRON_BF16_REVISION = "ce38b6ab8b252b4b8ee7165b4605e93191cafd73";
|
|
562
563
|
var MUSE_GLIMMER_BF16_REVISION = "a4e59da52a7bc87ae7251dd5545c0dd437c44b68";
|
|
563
564
|
var TRAINING_MODELS = [
|
|
564
565
|
{
|
|
565
566
|
id: "Qwen/Qwen3.5-2B",
|
|
566
567
|
family: "qwen3_5",
|
|
568
|
+
defaultRevision: QWEN_3_5_2B_REVISION,
|
|
567
569
|
modelLoader: "causal_lm",
|
|
568
570
|
defaultLearningRate: 1e-5,
|
|
569
571
|
defaultPerDeviceBatchSize: 1,
|
|
@@ -1079,508 +1081,24 @@ var localRunnerConfigSchema = z.object({
|
|
|
1079
1081
|
})
|
|
1080
1082
|
}).strict();
|
|
1081
1083
|
|
|
1082
|
-
// src/local-runtime/store.ts
|
|
1083
|
-
import { appendFile, copyFile as copyFile2, mkdir as mkdir2, readdir as readdir3, readFile as readFile2, rename as rename2, rm, stat as stat2, writeFile as writeFile2 } from "fs/promises";
|
|
1084
|
-
import { homedir } from "os";
|
|
1085
|
-
import { dirname as dirname2, join as join2, resolve as resolve3, sep as sep2 } from "path";
|
|
1086
|
-
import { randomUUID as randomUUID2 } from "crypto";
|
|
1087
|
-
function isTerminalRunState(state) {
|
|
1088
|
-
return state.status === "stage_completed" || state.status === "completed" || state.status === "failed" || state.status === "cancelled" && state.current_stage === "cancelled" && Boolean(state.completed_at);
|
|
1089
|
-
}
|
|
1090
|
-
function defaultLocalHome() {
|
|
1091
|
-
return resolve3(process.env.TT_LOCAL_HOME ?? join2(homedir(), ".tuned-tensor-local"));
|
|
1092
|
-
}
|
|
1093
|
-
function localStorePaths(root) {
|
|
1094
|
-
return {
|
|
1095
|
-
root,
|
|
1096
|
-
specsDir: join2(root, "specs"),
|
|
1097
|
-
runsDir: join2(root, "runs"),
|
|
1098
|
-
modelsDir: join2(root, "models"),
|
|
1099
|
-
datasetsDir: join2(root, "datasets")
|
|
1100
|
-
};
|
|
1101
|
-
}
|
|
1102
|
-
async function exists(path) {
|
|
1103
|
-
try {
|
|
1104
|
-
await stat2(path);
|
|
1105
|
-
return true;
|
|
1106
|
-
} catch {
|
|
1107
|
-
return false;
|
|
1108
|
-
}
|
|
1109
|
-
}
|
|
1110
|
-
async function writeJsonAtomic2(path, value) {
|
|
1111
|
-
await mkdir2(dirname2(path), { recursive: true });
|
|
1112
|
-
const tmp = `${path}.${process.pid}.${Date.now()}.${randomUUID2()}.tmp`;
|
|
1113
|
-
await writeFile2(tmp, `${JSON.stringify(value, null, 2)}
|
|
1114
|
-
`, "utf8");
|
|
1115
|
-
await rename2(tmp, path);
|
|
1116
|
-
}
|
|
1117
|
-
async function readJson2(path) {
|
|
1118
|
-
return JSON.parse(await readFile2(path, "utf8"));
|
|
1119
|
-
}
|
|
1120
|
-
async function appendJsonl(path, value) {
|
|
1121
|
-
await mkdir2(dirname2(path), { recursive: true });
|
|
1122
|
-
await appendFile(path, `${JSON.stringify(value)}
|
|
1123
|
-
`, "utf8");
|
|
1124
|
-
}
|
|
1125
|
-
async function readJsonl(path) {
|
|
1126
|
-
if (!await exists(path)) return [];
|
|
1127
|
-
return (await readFile2(path, "utf8")).split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
|
|
1128
|
-
}
|
|
1129
|
-
async function copyIfExists(from, to) {
|
|
1130
|
-
if (!await exists(from)) return;
|
|
1131
|
-
await mkdir2(dirname2(to), { recursive: true });
|
|
1132
|
-
await copyFile2(from, to);
|
|
1133
|
-
}
|
|
1134
|
-
async function childDirectoryNames(path) {
|
|
1135
|
-
try {
|
|
1136
|
-
const entries = await readdir3(path, { withFileTypes: true });
|
|
1137
|
-
return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
1138
|
-
} catch (error) {
|
|
1139
|
-
if (error.code === "ENOENT") return [];
|
|
1140
|
-
throw error;
|
|
1141
|
-
}
|
|
1142
|
-
}
|
|
1143
|
-
function compareIds(left, right) {
|
|
1144
|
-
if (left.id === right.id) return 0;
|
|
1145
|
-
return left.id < right.id ? -1 : 1;
|
|
1146
|
-
}
|
|
1147
|
-
function compareUpdatedAt(left, right) {
|
|
1148
|
-
if (left.updated_at !== right.updated_at) return left.updated_at > right.updated_at ? -1 : 1;
|
|
1149
|
-
return compareIds(left, right);
|
|
1150
|
-
}
|
|
1151
|
-
function compareCreatedAt(left, right) {
|
|
1152
|
-
if (left.created_at !== right.created_at) return left.created_at > right.created_at ? -1 : 1;
|
|
1153
|
-
return compareIds(left, right);
|
|
1154
|
-
}
|
|
1155
|
-
function findByIdOrPrefix(records, id) {
|
|
1156
|
-
return records.find((record) => record.id === id) ?? records.find((record) => record.id.startsWith(id));
|
|
1157
|
-
}
|
|
1158
|
-
function isPathInside(root, candidate) {
|
|
1159
|
-
const resolvedRoot = resolve3(root);
|
|
1160
|
-
const resolvedCandidate = resolve3(candidate);
|
|
1161
|
-
return resolvedCandidate === resolvedRoot || resolvedCandidate.startsWith(`${resolvedRoot}${sep2}`);
|
|
1162
|
-
}
|
|
1163
|
-
function createLocalStore(root = defaultLocalHome()) {
|
|
1164
|
-
const resolvedRoot = resolve3(root);
|
|
1165
|
-
const paths = localStorePaths(resolvedRoot);
|
|
1166
|
-
const runDir = (id) => join2(paths.runsDir, id);
|
|
1167
|
-
const runStatePath = (id) => join2(runDir(id), "state.json");
|
|
1168
|
-
const runEventsPath = (id) => join2(runDir(id), "progress.jsonl");
|
|
1169
|
-
const runRequestPath = (id) => join2(runDir(id), "request.json");
|
|
1170
|
-
const runReportPath = (id) => join2(runDir(id), "run-report.json");
|
|
1171
|
-
const cancellationPath = (id) => join2(runDir(id), "cancel.requested");
|
|
1172
|
-
const specDir = (id) => join2(paths.specsDir, id);
|
|
1173
|
-
const specPath = (id) => join2(specDir(id), "spec.json");
|
|
1174
|
-
const modelPath = (id) => join2(paths.modelsDir, id, "model.json");
|
|
1175
|
-
async function ensure() {
|
|
1176
|
-
await Promise.all([
|
|
1177
|
-
mkdir2(paths.specsDir, { recursive: true }),
|
|
1178
|
-
mkdir2(paths.runsDir, { recursive: true }),
|
|
1179
|
-
mkdir2(paths.modelsDir, { recursive: true }),
|
|
1180
|
-
mkdir2(paths.datasetsDir, { recursive: true })
|
|
1181
|
-
]);
|
|
1182
|
-
}
|
|
1183
|
-
async function writeRunState(state) {
|
|
1184
|
-
await writeJsonAtomic2(runStatePath(state.id), state);
|
|
1185
|
-
return state;
|
|
1186
|
-
}
|
|
1187
|
-
async function readSpecRecord(id) {
|
|
1188
|
-
const path = specPath(id);
|
|
1189
|
-
if (!await exists(path)) return null;
|
|
1190
|
-
const [spec, fileStats, directoryStats] = await Promise.all([
|
|
1191
|
-
readJson2(path),
|
|
1192
|
-
stat2(path),
|
|
1193
|
-
stat2(specDir(id))
|
|
1194
|
-
]);
|
|
1195
|
-
const createdAt = directoryStats.birthtimeMs > 0 ? directoryStats.birthtime : directoryStats.ctime;
|
|
1196
|
-
return {
|
|
1197
|
-
id,
|
|
1198
|
-
name: spec.name,
|
|
1199
|
-
base_model: spec.base_model,
|
|
1200
|
-
path,
|
|
1201
|
-
created_at: createdAt.toISOString(),
|
|
1202
|
-
updated_at: fileStats.mtime.toISOString()
|
|
1203
|
-
};
|
|
1204
|
-
}
|
|
1205
|
-
async function listSpecRecords() {
|
|
1206
|
-
await ensure();
|
|
1207
|
-
const records = [];
|
|
1208
|
-
for (const id of await childDirectoryNames(paths.specsDir)) {
|
|
1209
|
-
const record = await readSpecRecord(id);
|
|
1210
|
-
if (record) records.push(record);
|
|
1211
|
-
}
|
|
1212
|
-
return records.sort(compareUpdatedAt);
|
|
1213
|
-
}
|
|
1214
|
-
async function listRunRecords() {
|
|
1215
|
-
await ensure();
|
|
1216
|
-
const records = [];
|
|
1217
|
-
for (const id of await childDirectoryNames(paths.runsDir)) {
|
|
1218
|
-
const path = runStatePath(id);
|
|
1219
|
-
if (await exists(path)) records.push(await readJson2(path));
|
|
1220
|
-
}
|
|
1221
|
-
return records.sort(compareUpdatedAt);
|
|
1222
|
-
}
|
|
1223
|
-
async function listModelRecords() {
|
|
1224
|
-
await ensure();
|
|
1225
|
-
const records = [];
|
|
1226
|
-
for (const id of await childDirectoryNames(paths.modelsDir)) {
|
|
1227
|
-
const path = modelPath(id);
|
|
1228
|
-
if (await exists(path)) records.push(await readJson2(path));
|
|
1229
|
-
}
|
|
1230
|
-
return records.sort(compareCreatedAt);
|
|
1231
|
-
}
|
|
1232
|
-
async function importSpecRecord(specId, spec) {
|
|
1233
|
-
await ensure();
|
|
1234
|
-
await mkdir2(specDir(specId), { recursive: true });
|
|
1235
|
-
await writeJsonAtomic2(specPath(specId), spec);
|
|
1236
|
-
const record = await readSpecRecord(specId);
|
|
1237
|
-
if (!record) throw new Error(`Failed to persist spec: ${specId}`);
|
|
1238
|
-
return record;
|
|
1239
|
-
}
|
|
1240
|
-
async function appendRunEvent(state, event) {
|
|
1241
|
-
const row = {
|
|
1242
|
-
id: randomUUID2(),
|
|
1243
|
-
run_id: state.id,
|
|
1244
|
-
occurred_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1245
|
-
...event
|
|
1246
|
-
};
|
|
1247
|
-
await appendJsonl(runEventsPath(state.id), row);
|
|
1248
|
-
await appendJsonl(join2(state.artifact_dir, "progress.jsonl"), {
|
|
1249
|
-
at: row.occurred_at,
|
|
1250
|
-
stage: row.stage,
|
|
1251
|
-
status: row.status,
|
|
1252
|
-
message: row.message,
|
|
1253
|
-
...row.details ? { details: row.details } : {}
|
|
1254
|
-
});
|
|
1255
|
-
}
|
|
1256
|
-
async function finalizeCancellationState(runId) {
|
|
1257
|
-
const previous = await getRun(runId);
|
|
1258
|
-
if (previous.status === "cancelled" && previous.current_stage === "cancelled" && previous.completed_at) {
|
|
1259
|
-
return previous;
|
|
1260
|
-
}
|
|
1261
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1262
|
-
const cancelled = {
|
|
1263
|
-
...previous,
|
|
1264
|
-
status: "cancelled",
|
|
1265
|
-
current_stage: "cancelled",
|
|
1266
|
-
status_message: "Run cancelled.",
|
|
1267
|
-
error: void 0,
|
|
1268
|
-
completed_at: now,
|
|
1269
|
-
updated_at: now
|
|
1270
|
-
};
|
|
1271
|
-
await writeRunState(cancelled);
|
|
1272
|
-
await appendRunEvent(cancelled, { stage: "cancelled", status: "cancelled", message: "Run cancelled." });
|
|
1273
|
-
return cancelled;
|
|
1274
|
-
}
|
|
1275
|
-
async function preserveCancellationRequestState(runId) {
|
|
1276
|
-
const previous = await getRun(runId);
|
|
1277
|
-
if (isTerminalRunState(previous)) return previous;
|
|
1278
|
-
if (previous.status === "cancelled" && previous.current_stage === "cancel_requested") return previous;
|
|
1279
|
-
const requested = {
|
|
1280
|
-
...previous,
|
|
1281
|
-
status: "cancelled",
|
|
1282
|
-
current_stage: "cancel_requested",
|
|
1283
|
-
status_message: "Cancellation requested; waiting for the active worker to stop.",
|
|
1284
|
-
error: void 0,
|
|
1285
|
-
completed_at: void 0,
|
|
1286
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1287
|
-
};
|
|
1288
|
-
return writeRunState(requested);
|
|
1289
|
-
}
|
|
1290
|
-
async function resolveRunId(id) {
|
|
1291
|
-
await ensure();
|
|
1292
|
-
if (await exists(runStatePath(id))) return id;
|
|
1293
|
-
const record = findByIdOrPrefix(await listRunRecords(), id);
|
|
1294
|
-
if (!record) throw new Error(`Run not found: ${id}`);
|
|
1295
|
-
return record.id;
|
|
1296
|
-
}
|
|
1297
|
-
async function getRun(id) {
|
|
1298
|
-
return readJson2(runStatePath(await resolveRunId(id)));
|
|
1299
|
-
}
|
|
1300
|
-
async function registerModelRecord(args) {
|
|
1301
|
-
if (!args.training.model_artifact_uri || args.training.metrics?.dry_run === true) return null;
|
|
1302
|
-
const previous = await getRun(args.request.run_id);
|
|
1303
|
-
const modelId = `local-${args.request.run_id}`;
|
|
1304
|
-
const existingModel = await exists(modelPath(modelId)) ? await readJson2(modelPath(modelId)) : void 0;
|
|
1305
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1306
|
-
const model = {
|
|
1307
|
-
id: modelId,
|
|
1308
|
-
run_id: args.request.run_id,
|
|
1309
|
-
behavior_spec_id: args.request.behavior_spec_id,
|
|
1310
|
-
name: `${args.request.spec_snapshot.base_model} (${args.request.run_id.slice(0, 8)})`,
|
|
1311
|
-
provider: args.training.provider,
|
|
1312
|
-
base_model: args.request.spec_snapshot.base_model,
|
|
1313
|
-
artifact_uri: args.training.model_artifact_uri,
|
|
1314
|
-
artifact_dir: args.artifactDir,
|
|
1315
|
-
metrics: args.training.metrics,
|
|
1316
|
-
created_at: existingModel?.created_at ?? args.createdAt ?? now
|
|
1317
|
-
};
|
|
1318
|
-
await writeJsonAtomic2(modelPath(model.id), model);
|
|
1319
|
-
const alreadyRegistered = previous.model_id === modelId;
|
|
1320
|
-
const state = {
|
|
1321
|
-
...previous,
|
|
1322
|
-
model_id: modelId,
|
|
1323
|
-
updated_at: now
|
|
1324
|
-
};
|
|
1325
|
-
await writeRunState(state);
|
|
1326
|
-
if (!alreadyRegistered) {
|
|
1327
|
-
await appendRunEvent(state, {
|
|
1328
|
-
stage: "model_registered",
|
|
1329
|
-
status: "running",
|
|
1330
|
-
message: "Local model artifact registered.",
|
|
1331
|
-
details: { model_id: modelId, artifact_uri: model.artifact_uri }
|
|
1332
|
-
});
|
|
1333
|
-
}
|
|
1334
|
-
return model;
|
|
1335
|
-
}
|
|
1336
|
-
return {
|
|
1337
|
-
root: resolvedRoot,
|
|
1338
|
-
paths,
|
|
1339
|
-
ensure,
|
|
1340
|
-
importSpec: importSpecRecord,
|
|
1341
|
-
async listSpecs() {
|
|
1342
|
-
return listSpecRecords();
|
|
1343
|
-
},
|
|
1344
|
-
async getSpec(id) {
|
|
1345
|
-
const record = findByIdOrPrefix(await listSpecRecords(), id);
|
|
1346
|
-
if (!record) throw new Error(`Spec not found: ${id}`);
|
|
1347
|
-
return { ...record, spec: await readJson2(record.path) };
|
|
1348
|
-
},
|
|
1349
|
-
async startRun({ request, artifactDir }) {
|
|
1350
|
-
await ensure();
|
|
1351
|
-
await importSpecRecord(request.behavior_spec_id, request.spec_snapshot);
|
|
1352
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1353
|
-
const state = {
|
|
1354
|
-
id: request.run_id,
|
|
1355
|
-
behavior_spec_id: request.behavior_spec_id,
|
|
1356
|
-
user_id: request.user_id,
|
|
1357
|
-
run_number: request.run_number,
|
|
1358
|
-
status: "queued",
|
|
1359
|
-
current_stage: "queued",
|
|
1360
|
-
status_message: "Run queued.",
|
|
1361
|
-
artifact_dir: artifactDir,
|
|
1362
|
-
base_model: request.spec_snapshot.base_model,
|
|
1363
|
-
spec_name: request.spec_snapshot.name,
|
|
1364
|
-
created_at: now,
|
|
1365
|
-
updated_at: now,
|
|
1366
|
-
started_at: now
|
|
1367
|
-
};
|
|
1368
|
-
await writeJsonAtomic2(runRequestPath(request.run_id), request);
|
|
1369
|
-
await writeRunState(state);
|
|
1370
|
-
await appendRunEvent(state, { stage: "queued", status: "queued", message: "Run queued." });
|
|
1371
|
-
return state;
|
|
1372
|
-
},
|
|
1373
|
-
async syncRunRequest(request, artifactDir) {
|
|
1374
|
-
const previous = await getRun(request.run_id);
|
|
1375
|
-
if (previous.behavior_spec_id !== request.behavior_spec_id || previous.user_id !== request.user_id || previous.run_number !== request.run_number) {
|
|
1376
|
-
throw new Error(
|
|
1377
|
-
`Run ${request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
|
|
1378
|
-
);
|
|
1379
|
-
}
|
|
1380
|
-
await importSpecRecord(request.behavior_spec_id, request.spec_snapshot);
|
|
1381
|
-
await writeJsonAtomic2(runRequestPath(request.run_id), request);
|
|
1382
|
-
const state = {
|
|
1383
|
-
...previous,
|
|
1384
|
-
artifact_dir: artifactDir ?? previous.artifact_dir,
|
|
1385
|
-
base_model: request.spec_snapshot.base_model,
|
|
1386
|
-
spec_name: request.spec_snapshot.name,
|
|
1387
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1388
|
-
};
|
|
1389
|
-
await writeRunState(state);
|
|
1390
|
-
return state;
|
|
1391
|
-
},
|
|
1392
|
-
async updateRun({ runId, status, stage, message, details }) {
|
|
1393
|
-
const previous = await getRun(runId);
|
|
1394
|
-
if (status !== "cancelled" && await exists(cancellationPath(previous.id))) {
|
|
1395
|
-
return preserveCancellationRequestState(previous.id);
|
|
1396
|
-
}
|
|
1397
|
-
const active = status !== "stage_completed" && status !== "completed" && status !== "failed" && status !== "cancelled";
|
|
1398
|
-
const successful = status === "stage_completed" || status === "completed";
|
|
1399
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1400
|
-
const state = {
|
|
1401
|
-
...previous,
|
|
1402
|
-
status,
|
|
1403
|
-
current_stage: stage,
|
|
1404
|
-
status_message: message,
|
|
1405
|
-
...active ? { error: void 0, completed_at: void 0 } : successful ? { error: void 0, completed_at: now } : {},
|
|
1406
|
-
updated_at: now
|
|
1407
|
-
};
|
|
1408
|
-
await writeRunState(state);
|
|
1409
|
-
if (status !== "cancelled" && await exists(cancellationPath(previous.id))) {
|
|
1410
|
-
return preserveCancellationRequestState(previous.id);
|
|
1411
|
-
}
|
|
1412
|
-
await appendRunEvent(state, {
|
|
1413
|
-
stage,
|
|
1414
|
-
status: successful ? "completed" : status === "failed" ? "failed" : "running",
|
|
1415
|
-
message,
|
|
1416
|
-
details
|
|
1417
|
-
});
|
|
1418
|
-
return state;
|
|
1419
|
-
},
|
|
1420
|
-
async completeRun(report, artifactDir, reportPath) {
|
|
1421
|
-
if (await exists(cancellationPath(report.run_id))) {
|
|
1422
|
-
return preserveCancellationRequestState(report.run_id);
|
|
1423
|
-
}
|
|
1424
|
-
const request = await readJson2(runRequestPath(report.run_id));
|
|
1425
|
-
const completedAt = report.created_at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
1426
|
-
const model = await registerModelRecord({
|
|
1427
|
-
request,
|
|
1428
|
-
training: report.training,
|
|
1429
|
-
artifactDir,
|
|
1430
|
-
createdAt: completedAt
|
|
1431
|
-
});
|
|
1432
|
-
if (!model) {
|
|
1433
|
-
await rm(join2(paths.modelsDir, `local-${report.run_id}`), { recursive: true, force: true });
|
|
1434
|
-
}
|
|
1435
|
-
const previous = await getRun(report.run_id);
|
|
1436
|
-
const state = {
|
|
1437
|
-
...previous,
|
|
1438
|
-
status: "completed",
|
|
1439
|
-
current_stage: "completed",
|
|
1440
|
-
status_message: "Run completed successfully.",
|
|
1441
|
-
report_path: reportPath,
|
|
1442
|
-
model_id: model?.id,
|
|
1443
|
-
error: void 0,
|
|
1444
|
-
completed_at: completedAt,
|
|
1445
|
-
updated_at: completedAt
|
|
1446
|
-
};
|
|
1447
|
-
await writeRunState(state);
|
|
1448
|
-
if (await exists(cancellationPath(report.run_id))) {
|
|
1449
|
-
return preserveCancellationRequestState(report.run_id);
|
|
1450
|
-
}
|
|
1451
|
-
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
1452
|
-
await appendRunEvent(state, {
|
|
1453
|
-
stage: "completed",
|
|
1454
|
-
status: "completed",
|
|
1455
|
-
message: "Run completed successfully.",
|
|
1456
|
-
details: { report_path: reportPath, ...model ? { model_id: model.id } : {} }
|
|
1457
|
-
});
|
|
1458
|
-
return state;
|
|
1459
|
-
},
|
|
1460
|
-
registerModel: registerModelRecord,
|
|
1461
|
-
async invalidateRunOutputs(runId, options) {
|
|
1462
|
-
const previous = await getRun(runId);
|
|
1463
|
-
if (options.report) await rm(runReportPath(previous.id), { force: true });
|
|
1464
|
-
if (options.model) {
|
|
1465
|
-
await rm(join2(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true });
|
|
1466
|
-
}
|
|
1467
|
-
const state = {
|
|
1468
|
-
...previous,
|
|
1469
|
-
...options.report ? { report_path: void 0 } : {},
|
|
1470
|
-
...options.model ? { model_id: void 0 } : {},
|
|
1471
|
-
error: void 0,
|
|
1472
|
-
completed_at: void 0,
|
|
1473
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1474
|
-
};
|
|
1475
|
-
return writeRunState(state);
|
|
1476
|
-
},
|
|
1477
|
-
async failRun(runId, error) {
|
|
1478
|
-
const previous = await getRun(runId);
|
|
1479
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1480
|
-
if (await exists(cancellationPath(previous.id))) {
|
|
1481
|
-
return preserveCancellationRequestState(previous.id);
|
|
1482
|
-
}
|
|
1483
|
-
const state = {
|
|
1484
|
-
...previous,
|
|
1485
|
-
status: "failed",
|
|
1486
|
-
current_stage: "failed",
|
|
1487
|
-
status_message: error,
|
|
1488
|
-
error,
|
|
1489
|
-
completed_at: now,
|
|
1490
|
-
updated_at: now
|
|
1491
|
-
};
|
|
1492
|
-
await writeRunState(state);
|
|
1493
|
-
if (await exists(cancellationPath(previous.id))) {
|
|
1494
|
-
return preserveCancellationRequestState(previous.id);
|
|
1495
|
-
}
|
|
1496
|
-
await appendRunEvent(state, { stage: "failed", status: "failed", message: error });
|
|
1497
|
-
return state;
|
|
1498
|
-
},
|
|
1499
|
-
async cancelRun(runId) {
|
|
1500
|
-
const state = await getRun(runId);
|
|
1501
|
-
if (isTerminalRunState(state) || state.status === "cancelled" && state.current_stage === "cancel_requested") {
|
|
1502
|
-
return;
|
|
1503
|
-
}
|
|
1504
|
-
await writeFile2(cancellationPath(state.id), `${(/* @__PURE__ */ new Date()).toISOString()}
|
|
1505
|
-
`, "utf8");
|
|
1506
|
-
const latest = await getRun(state.id);
|
|
1507
|
-
if (isTerminalRunState(latest)) {
|
|
1508
|
-
if (latest.status !== "cancelled") await rm(cancellationPath(state.id), { force: true });
|
|
1509
|
-
return;
|
|
1510
|
-
}
|
|
1511
|
-
const updated = {
|
|
1512
|
-
...latest,
|
|
1513
|
-
status: "cancelled",
|
|
1514
|
-
current_stage: "cancel_requested",
|
|
1515
|
-
status_message: "Cancellation requested; waiting for the active worker to stop.",
|
|
1516
|
-
error: void 0,
|
|
1517
|
-
completed_at: void 0,
|
|
1518
|
-
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1519
|
-
};
|
|
1520
|
-
await writeRunState(updated);
|
|
1521
|
-
await appendRunEvent(updated, { stage: "cancel_requested", status: "cancelled", message: "Cancellation requested." });
|
|
1522
|
-
},
|
|
1523
|
-
async finalizeCancellation(runId) {
|
|
1524
|
-
return finalizeCancellationState(runId);
|
|
1525
|
-
},
|
|
1526
|
-
async isCancellationRequested(runId) {
|
|
1527
|
-
const state = await getRun(runId);
|
|
1528
|
-
return state.status === "cancelled" || await exists(cancellationPath(state.id));
|
|
1529
|
-
},
|
|
1530
|
-
async listRuns() {
|
|
1531
|
-
return listRunRecords();
|
|
1532
|
-
},
|
|
1533
|
-
getRun,
|
|
1534
|
-
async getRunEvents(id) {
|
|
1535
|
-
const state = await getRun(id);
|
|
1536
|
-
return readJsonl(runEventsPath(state.id));
|
|
1537
|
-
},
|
|
1538
|
-
async getRunReport(id) {
|
|
1539
|
-
const state = await getRun(id);
|
|
1540
|
-
const candidates = [];
|
|
1541
|
-
if (state.report_path) {
|
|
1542
|
-
const reportPath = resolve3(state.report_path);
|
|
1543
|
-
const allowedRoots = [resolvedRoot];
|
|
1544
|
-
if (state.artifact_dir) allowedRoots.push(resolve3(state.artifact_dir));
|
|
1545
|
-
if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) {
|
|
1546
|
-
candidates.push(reportPath);
|
|
1547
|
-
}
|
|
1548
|
-
}
|
|
1549
|
-
candidates.push(runReportPath(state.id));
|
|
1550
|
-
for (const path of candidates) {
|
|
1551
|
-
if (await exists(path)) return readJson2(path);
|
|
1552
|
-
}
|
|
1553
|
-
throw new Error(`Run has no report yet: ${id}`);
|
|
1554
|
-
},
|
|
1555
|
-
async listModels() {
|
|
1556
|
-
return listModelRecords();
|
|
1557
|
-
},
|
|
1558
|
-
async getModel(id) {
|
|
1559
|
-
const record = findByIdOrPrefix(await listModelRecords(), id);
|
|
1560
|
-
if (!record) throw new Error(`Model not found: ${id}`);
|
|
1561
|
-
return record;
|
|
1562
|
-
}
|
|
1563
|
-
};
|
|
1564
|
-
}
|
|
1565
|
-
|
|
1566
1084
|
// src/local-runtime/local-project.ts
|
|
1567
|
-
import { createHash as createHash2, randomUUID as
|
|
1568
|
-
import { readFile as
|
|
1569
|
-
import { dirname as
|
|
1570
|
-
import { homedir
|
|
1571
|
-
import { mkdir as
|
|
1085
|
+
import { createHash as createHash2, randomUUID as randomUUID2 } from "crypto";
|
|
1086
|
+
import { readFile as readFile2, stat as stat2, writeFile as writeFile2 } from "fs/promises";
|
|
1087
|
+
import { dirname as dirname2, isAbsolute as isAbsolute2, resolve as resolve3 } from "path";
|
|
1088
|
+
import { homedir } from "os";
|
|
1089
|
+
import { mkdir as mkdir2 } from "fs/promises";
|
|
1572
1090
|
var DEFAULT_LOCAL_SPEC_PATH = "tunedtensor.json";
|
|
1573
1091
|
function resolveLocalReference(value, baseDirectory) {
|
|
1574
1092
|
if (typeof value !== "string" || !value || isAbsolute2(value) || /^[a-z][a-z0-9+.-]*:/i.test(value)) {
|
|
1575
1093
|
return value;
|
|
1576
1094
|
}
|
|
1577
|
-
if (value === "~") return
|
|
1578
|
-
if (value.startsWith("~/")) return
|
|
1579
|
-
return
|
|
1095
|
+
if (value === "~") return homedir();
|
|
1096
|
+
if (value.startsWith("~/")) return resolve3(homedir(), value.slice(2));
|
|
1097
|
+
return resolve3(baseDirectory, value);
|
|
1580
1098
|
}
|
|
1581
1099
|
function resolveLocalRunInputPaths(raw, inputPath) {
|
|
1582
1100
|
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw;
|
|
1583
|
-
const baseDirectory =
|
|
1101
|
+
const baseDirectory = dirname2(resolve3(inputPath));
|
|
1584
1102
|
const value = structuredClone(raw);
|
|
1585
1103
|
const dataset = value.dataset_prebuilt;
|
|
1586
1104
|
if (dataset && typeof dataset === "object" && !Array.isArray(dataset)) {
|
|
@@ -1591,20 +1109,20 @@ function resolveLocalRunInputPaths(raw, inputPath) {
|
|
|
1591
1109
|
}
|
|
1592
1110
|
return value;
|
|
1593
1111
|
}
|
|
1594
|
-
async function
|
|
1112
|
+
async function exists(path) {
|
|
1595
1113
|
try {
|
|
1596
|
-
await
|
|
1114
|
+
await stat2(path);
|
|
1597
1115
|
return true;
|
|
1598
1116
|
} catch {
|
|
1599
1117
|
return false;
|
|
1600
1118
|
}
|
|
1601
1119
|
}
|
|
1602
1120
|
async function initLocalSpecFile(args) {
|
|
1603
|
-
if (!args.force && await
|
|
1121
|
+
if (!args.force && await exists(args.outputPath)) {
|
|
1604
1122
|
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
1605
1123
|
}
|
|
1606
1124
|
const spec = localBehaviorSpecFileSchema.parse({
|
|
1607
|
-
id:
|
|
1125
|
+
id: randomUUID2(),
|
|
1608
1126
|
name: args.name,
|
|
1609
1127
|
description: "",
|
|
1610
1128
|
system_prompt: "Describe the behavior this local model should learn.",
|
|
@@ -1627,13 +1145,13 @@ async function initLocalSpecFile(args) {
|
|
|
1627
1145
|
n_epochs: 1
|
|
1628
1146
|
}
|
|
1629
1147
|
});
|
|
1630
|
-
await
|
|
1631
|
-
await
|
|
1148
|
+
await mkdir2(dirname2(args.outputPath), { recursive: true });
|
|
1149
|
+
await writeFile2(args.outputPath, `${JSON.stringify(spec, null, 2)}
|
|
1632
1150
|
`, "utf8");
|
|
1633
1151
|
return spec;
|
|
1634
1152
|
}
|
|
1635
1153
|
async function initLocalRunnerConfigFile(args) {
|
|
1636
|
-
if (!args.force && await
|
|
1154
|
+
if (!args.force && await exists(args.outputPath)) {
|
|
1637
1155
|
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
1638
1156
|
}
|
|
1639
1157
|
const config = {
|
|
@@ -1649,8 +1167,8 @@ async function initLocalRunnerConfigFile(args) {
|
|
|
1649
1167
|
timeoutMs: 18e5
|
|
1650
1168
|
}
|
|
1651
1169
|
};
|
|
1652
|
-
await
|
|
1653
|
-
await
|
|
1170
|
+
await mkdir2(dirname2(args.outputPath), { recursive: true });
|
|
1171
|
+
await writeFile2(args.outputPath, `${JSON.stringify(config, null, 2)}
|
|
1654
1172
|
`, "utf8");
|
|
1655
1173
|
return config;
|
|
1656
1174
|
}
|
|
@@ -1681,7 +1199,7 @@ function runRequestFromLocalSpec(spec, options = {}) {
|
|
|
1681
1199
|
} = spec;
|
|
1682
1200
|
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
1683
1201
|
return fineTuneRunRequestSchema.parse({
|
|
1684
|
-
run_id: options.runId ??
|
|
1202
|
+
run_id: options.runId ?? randomUUID2(),
|
|
1685
1203
|
user_id: "local-user",
|
|
1686
1204
|
behavior_spec_id: behaviorSpecId,
|
|
1687
1205
|
run_number: 1,
|
|
@@ -1705,7 +1223,7 @@ function deterministicBehaviorSpecId(spec) {
|
|
|
1705
1223
|
].join("-");
|
|
1706
1224
|
}
|
|
1707
1225
|
async function loadLocalRunInput(path, options = {}) {
|
|
1708
|
-
const raw = resolveLocalRunInputPaths(JSON.parse(await
|
|
1226
|
+
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile2(path, "utf8")), path);
|
|
1709
1227
|
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
1710
1228
|
if (request.success) {
|
|
1711
1229
|
return { kind: "request", path, request: request.data };
|
|
@@ -1720,9 +1238,9 @@ async function loadLocalRunInput(path, options = {}) {
|
|
|
1720
1238
|
}
|
|
1721
1239
|
|
|
1722
1240
|
// src/local-runtime/dataset.ts
|
|
1723
|
-
import { readFile as
|
|
1241
|
+
import { readFile as readFile3 } from "fs/promises";
|
|
1724
1242
|
async function loadNormalizedChatJsonl(path) {
|
|
1725
|
-
const text = await
|
|
1243
|
+
const text = await readFile3(path, "utf8");
|
|
1726
1244
|
const rows = [];
|
|
1727
1245
|
const jsonLines = [];
|
|
1728
1246
|
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
@@ -1859,29 +1377,513 @@ function forwardStreamLines(stream, onLine) {
|
|
|
1859
1377
|
for (const line of lines) {
|
|
1860
1378
|
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
1861
1379
|
}
|
|
1862
|
-
});
|
|
1863
|
-
stream.on("end", () => {
|
|
1864
|
-
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
1865
|
-
});
|
|
1866
|
-
}
|
|
1867
|
-
|
|
1868
|
-
// src/local-runtime/package-root.ts
|
|
1869
|
-
import { existsSync } from "fs";
|
|
1870
|
-
import { dirname as
|
|
1871
|
-
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
1872
|
-
function localRuntimePackageRoot(moduleUrl) {
|
|
1873
|
-
const moduleDirectory =
|
|
1874
|
-
const candidates = [
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
];
|
|
1878
|
-
const root = candidates.find(
|
|
1879
|
-
(candidate) => existsSync(
|
|
1880
|
-
);
|
|
1881
|
-
if (!root) {
|
|
1882
|
-
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
1380
|
+
});
|
|
1381
|
+
stream.on("end", () => {
|
|
1382
|
+
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
1383
|
+
});
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
// src/local-runtime/package-root.ts
|
|
1387
|
+
import { existsSync } from "fs";
|
|
1388
|
+
import { dirname as dirname3, resolve as resolve4 } from "path";
|
|
1389
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
1390
|
+
function localRuntimePackageRoot(moduleUrl) {
|
|
1391
|
+
const moduleDirectory = dirname3(fileURLToPath2(moduleUrl));
|
|
1392
|
+
const candidates = [
|
|
1393
|
+
dirname3(moduleDirectory),
|
|
1394
|
+
dirname3(dirname3(moduleDirectory))
|
|
1395
|
+
];
|
|
1396
|
+
const root = candidates.find(
|
|
1397
|
+
(candidate) => existsSync(resolve4(candidate, "package.json")) && existsSync(resolve4(candidate, "training/local-runner"))
|
|
1398
|
+
);
|
|
1399
|
+
if (!root) {
|
|
1400
|
+
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
1401
|
+
}
|
|
1402
|
+
return root;
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
// src/local-runtime/store.ts
|
|
1406
|
+
import { appendFile, copyFile as copyFile2, mkdir as mkdir3, readdir as readdir3, readFile as readFile4, rename as rename2, rm, stat as stat3, writeFile as writeFile3 } from "fs/promises";
|
|
1407
|
+
import { homedir as homedir2 } from "os";
|
|
1408
|
+
import { dirname as dirname4, join as join2, resolve as resolve5, sep as sep2 } from "path";
|
|
1409
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
1410
|
+
function isTerminalRunState(state) {
|
|
1411
|
+
return state.status === "stage_completed" || state.status === "completed" || state.status === "failed" || state.status === "cancelled" && state.current_stage === "cancelled" && Boolean(state.completed_at);
|
|
1412
|
+
}
|
|
1413
|
+
function defaultLocalHome() {
|
|
1414
|
+
return resolve5(process.env.TT_LOCAL_HOME ?? join2(homedir2(), ".tuned-tensor-local"));
|
|
1415
|
+
}
|
|
1416
|
+
function localStorePaths(root) {
|
|
1417
|
+
return {
|
|
1418
|
+
root,
|
|
1419
|
+
specsDir: join2(root, "specs"),
|
|
1420
|
+
runsDir: join2(root, "runs"),
|
|
1421
|
+
modelsDir: join2(root, "models"),
|
|
1422
|
+
datasetsDir: join2(root, "datasets")
|
|
1423
|
+
};
|
|
1424
|
+
}
|
|
1425
|
+
async function exists2(path) {
|
|
1426
|
+
try {
|
|
1427
|
+
await stat3(path);
|
|
1428
|
+
return true;
|
|
1429
|
+
} catch {
|
|
1430
|
+
return false;
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
async function writeJsonAtomic2(path, value) {
|
|
1434
|
+
await mkdir3(dirname4(path), { recursive: true });
|
|
1435
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.${randomUUID3()}.tmp`;
|
|
1436
|
+
await writeFile3(tmp, `${JSON.stringify(value, null, 2)}
|
|
1437
|
+
`, "utf8");
|
|
1438
|
+
await rename2(tmp, path);
|
|
1439
|
+
}
|
|
1440
|
+
async function readJson2(path) {
|
|
1441
|
+
return JSON.parse(await readFile4(path, "utf8"));
|
|
1442
|
+
}
|
|
1443
|
+
async function appendJsonl(path, value) {
|
|
1444
|
+
await mkdir3(dirname4(path), { recursive: true });
|
|
1445
|
+
await appendFile(path, `${JSON.stringify(value)}
|
|
1446
|
+
`, "utf8");
|
|
1447
|
+
}
|
|
1448
|
+
async function readJsonl(path) {
|
|
1449
|
+
if (!await exists2(path)) return [];
|
|
1450
|
+
return (await readFile4(path, "utf8")).split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
|
|
1451
|
+
}
|
|
1452
|
+
async function copyIfExists(from, to) {
|
|
1453
|
+
if (!await exists2(from)) return;
|
|
1454
|
+
await mkdir3(dirname4(to), { recursive: true });
|
|
1455
|
+
await copyFile2(from, to);
|
|
1456
|
+
}
|
|
1457
|
+
async function childDirectoryNames(path) {
|
|
1458
|
+
try {
|
|
1459
|
+
const entries = await readdir3(path, { withFileTypes: true });
|
|
1460
|
+
return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
1461
|
+
} catch (error) {
|
|
1462
|
+
if (error.code === "ENOENT") return [];
|
|
1463
|
+
throw error;
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
function compareIds(left, right) {
|
|
1467
|
+
if (left.id === right.id) return 0;
|
|
1468
|
+
return left.id < right.id ? -1 : 1;
|
|
1469
|
+
}
|
|
1470
|
+
function compareUpdatedAt(left, right) {
|
|
1471
|
+
if (left.updated_at !== right.updated_at) return left.updated_at > right.updated_at ? -1 : 1;
|
|
1472
|
+
return compareIds(left, right);
|
|
1473
|
+
}
|
|
1474
|
+
function compareCreatedAt(left, right) {
|
|
1475
|
+
if (left.created_at !== right.created_at) return left.created_at > right.created_at ? -1 : 1;
|
|
1476
|
+
return compareIds(left, right);
|
|
1477
|
+
}
|
|
1478
|
+
function findByIdOrPrefix(records, id) {
|
|
1479
|
+
return records.find((record) => record.id === id) ?? records.find((record) => record.id.startsWith(id));
|
|
1480
|
+
}
|
|
1481
|
+
function isPathInside(root, candidate) {
|
|
1482
|
+
const resolvedRoot = resolve5(root);
|
|
1483
|
+
const resolvedCandidate = resolve5(candidate);
|
|
1484
|
+
return resolvedCandidate === resolvedRoot || resolvedCandidate.startsWith(`${resolvedRoot}${sep2}`);
|
|
1485
|
+
}
|
|
1486
|
+
function createLocalStore(root = defaultLocalHome()) {
|
|
1487
|
+
const resolvedRoot = resolve5(root);
|
|
1488
|
+
const paths = localStorePaths(resolvedRoot);
|
|
1489
|
+
const runDir = (id) => join2(paths.runsDir, id);
|
|
1490
|
+
const runStatePath = (id) => join2(runDir(id), "state.json");
|
|
1491
|
+
const runEventsPath = (id) => join2(runDir(id), "progress.jsonl");
|
|
1492
|
+
const runRequestPath = (id) => join2(runDir(id), "request.json");
|
|
1493
|
+
const runReportPath = (id) => join2(runDir(id), "run-report.json");
|
|
1494
|
+
const cancellationPath = (id) => join2(runDir(id), "cancel.requested");
|
|
1495
|
+
const specDir = (id) => join2(paths.specsDir, id);
|
|
1496
|
+
const specPath = (id) => join2(specDir(id), "spec.json");
|
|
1497
|
+
const modelPath = (id) => join2(paths.modelsDir, id, "model.json");
|
|
1498
|
+
async function ensure() {
|
|
1499
|
+
await Promise.all([
|
|
1500
|
+
mkdir3(paths.specsDir, { recursive: true }),
|
|
1501
|
+
mkdir3(paths.runsDir, { recursive: true }),
|
|
1502
|
+
mkdir3(paths.modelsDir, { recursive: true }),
|
|
1503
|
+
mkdir3(paths.datasetsDir, { recursive: true })
|
|
1504
|
+
]);
|
|
1505
|
+
}
|
|
1506
|
+
async function writeRunState(state) {
|
|
1507
|
+
await writeJsonAtomic2(runStatePath(state.id), state);
|
|
1508
|
+
return state;
|
|
1509
|
+
}
|
|
1510
|
+
async function readSpecRecord(id) {
|
|
1511
|
+
const path = specPath(id);
|
|
1512
|
+
if (!await exists2(path)) return null;
|
|
1513
|
+
const [spec, fileStats, directoryStats] = await Promise.all([
|
|
1514
|
+
readJson2(path),
|
|
1515
|
+
stat3(path),
|
|
1516
|
+
stat3(specDir(id))
|
|
1517
|
+
]);
|
|
1518
|
+
const createdAt = directoryStats.birthtimeMs > 0 ? directoryStats.birthtime : directoryStats.ctime;
|
|
1519
|
+
return {
|
|
1520
|
+
id,
|
|
1521
|
+
name: spec.name,
|
|
1522
|
+
base_model: spec.base_model,
|
|
1523
|
+
path,
|
|
1524
|
+
created_at: createdAt.toISOString(),
|
|
1525
|
+
updated_at: fileStats.mtime.toISOString()
|
|
1526
|
+
};
|
|
1527
|
+
}
|
|
1528
|
+
async function listSpecRecords() {
|
|
1529
|
+
await ensure();
|
|
1530
|
+
const records = [];
|
|
1531
|
+
for (const id of await childDirectoryNames(paths.specsDir)) {
|
|
1532
|
+
const record = await readSpecRecord(id);
|
|
1533
|
+
if (record) records.push(record);
|
|
1534
|
+
}
|
|
1535
|
+
return records.sort(compareUpdatedAt);
|
|
1536
|
+
}
|
|
1537
|
+
async function listRunRecords() {
|
|
1538
|
+
await ensure();
|
|
1539
|
+
const records = [];
|
|
1540
|
+
for (const id of await childDirectoryNames(paths.runsDir)) {
|
|
1541
|
+
const path = runStatePath(id);
|
|
1542
|
+
if (await exists2(path)) records.push(await readJson2(path));
|
|
1543
|
+
}
|
|
1544
|
+
return records.sort(compareUpdatedAt);
|
|
1545
|
+
}
|
|
1546
|
+
async function listModelRecords() {
|
|
1547
|
+
await ensure();
|
|
1548
|
+
const records = [];
|
|
1549
|
+
for (const id of await childDirectoryNames(paths.modelsDir)) {
|
|
1550
|
+
const path = modelPath(id);
|
|
1551
|
+
if (await exists2(path)) records.push(await readJson2(path));
|
|
1552
|
+
}
|
|
1553
|
+
return records.sort(compareCreatedAt);
|
|
1554
|
+
}
|
|
1555
|
+
async function importSpecRecord(specId, spec) {
|
|
1556
|
+
await ensure();
|
|
1557
|
+
await mkdir3(specDir(specId), { recursive: true });
|
|
1558
|
+
await writeJsonAtomic2(specPath(specId), spec);
|
|
1559
|
+
const record = await readSpecRecord(specId);
|
|
1560
|
+
if (!record) throw new Error(`Failed to persist spec: ${specId}`);
|
|
1561
|
+
return record;
|
|
1562
|
+
}
|
|
1563
|
+
async function appendRunEvent(state, event) {
|
|
1564
|
+
const row = {
|
|
1565
|
+
id: randomUUID3(),
|
|
1566
|
+
run_id: state.id,
|
|
1567
|
+
occurred_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1568
|
+
...event
|
|
1569
|
+
};
|
|
1570
|
+
await appendJsonl(runEventsPath(state.id), row);
|
|
1571
|
+
await appendJsonl(join2(state.artifact_dir, "progress.jsonl"), {
|
|
1572
|
+
at: row.occurred_at,
|
|
1573
|
+
stage: row.stage,
|
|
1574
|
+
status: row.status,
|
|
1575
|
+
message: row.message,
|
|
1576
|
+
...row.details ? { details: row.details } : {}
|
|
1577
|
+
});
|
|
1578
|
+
}
|
|
1579
|
+
async function finalizeCancellationState(runId) {
|
|
1580
|
+
const previous = await getRun(runId);
|
|
1581
|
+
if (previous.status === "cancelled" && previous.current_stage === "cancelled" && previous.completed_at) {
|
|
1582
|
+
return previous;
|
|
1583
|
+
}
|
|
1584
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1585
|
+
const cancelled = {
|
|
1586
|
+
...previous,
|
|
1587
|
+
status: "cancelled",
|
|
1588
|
+
current_stage: "cancelled",
|
|
1589
|
+
status_message: "Run cancelled.",
|
|
1590
|
+
error: void 0,
|
|
1591
|
+
completed_at: now,
|
|
1592
|
+
updated_at: now
|
|
1593
|
+
};
|
|
1594
|
+
await writeRunState(cancelled);
|
|
1595
|
+
await appendRunEvent(cancelled, { stage: "cancelled", status: "cancelled", message: "Run cancelled." });
|
|
1596
|
+
return cancelled;
|
|
1597
|
+
}
|
|
1598
|
+
async function preserveCancellationRequestState(runId) {
|
|
1599
|
+
const previous = await getRun(runId);
|
|
1600
|
+
if (isTerminalRunState(previous)) return previous;
|
|
1601
|
+
if (previous.status === "cancelled" && previous.current_stage === "cancel_requested") return previous;
|
|
1602
|
+
const requested = {
|
|
1603
|
+
...previous,
|
|
1604
|
+
status: "cancelled",
|
|
1605
|
+
current_stage: "cancel_requested",
|
|
1606
|
+
status_message: "Cancellation requested; waiting for the active worker to stop.",
|
|
1607
|
+
error: void 0,
|
|
1608
|
+
completed_at: void 0,
|
|
1609
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1610
|
+
};
|
|
1611
|
+
return writeRunState(requested);
|
|
1612
|
+
}
|
|
1613
|
+
async function resolveRunId(id) {
|
|
1614
|
+
await ensure();
|
|
1615
|
+
if (await exists2(runStatePath(id))) return id;
|
|
1616
|
+
const record = findByIdOrPrefix(await listRunRecords(), id);
|
|
1617
|
+
if (!record) throw new Error(`Run not found: ${id}`);
|
|
1618
|
+
return record.id;
|
|
1619
|
+
}
|
|
1620
|
+
async function getRun(id) {
|
|
1621
|
+
return readJson2(runStatePath(await resolveRunId(id)));
|
|
1622
|
+
}
|
|
1623
|
+
async function registerModelRecord(args) {
|
|
1624
|
+
if (!args.training.model_artifact_uri || args.training.metrics?.dry_run === true) return null;
|
|
1625
|
+
const previous = await getRun(args.request.run_id);
|
|
1626
|
+
const modelId = `local-${args.request.run_id}`;
|
|
1627
|
+
const existingModel = await exists2(modelPath(modelId)) ? await readJson2(modelPath(modelId)) : void 0;
|
|
1628
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1629
|
+
const model = {
|
|
1630
|
+
id: modelId,
|
|
1631
|
+
run_id: args.request.run_id,
|
|
1632
|
+
behavior_spec_id: args.request.behavior_spec_id,
|
|
1633
|
+
name: `${args.request.spec_snapshot.base_model} (${args.request.run_id.slice(0, 8)})`,
|
|
1634
|
+
provider: args.training.provider,
|
|
1635
|
+
base_model: args.request.spec_snapshot.base_model,
|
|
1636
|
+
artifact_uri: args.training.model_artifact_uri,
|
|
1637
|
+
artifact_dir: args.artifactDir,
|
|
1638
|
+
metrics: args.training.metrics,
|
|
1639
|
+
created_at: existingModel?.created_at ?? args.createdAt ?? now
|
|
1640
|
+
};
|
|
1641
|
+
await writeJsonAtomic2(modelPath(model.id), model);
|
|
1642
|
+
const alreadyRegistered = previous.model_id === modelId;
|
|
1643
|
+
const state = {
|
|
1644
|
+
...previous,
|
|
1645
|
+
model_id: modelId,
|
|
1646
|
+
updated_at: now
|
|
1647
|
+
};
|
|
1648
|
+
await writeRunState(state);
|
|
1649
|
+
if (!alreadyRegistered) {
|
|
1650
|
+
await appendRunEvent(state, {
|
|
1651
|
+
stage: "model_registered",
|
|
1652
|
+
status: "running",
|
|
1653
|
+
message: "Local model artifact registered.",
|
|
1654
|
+
details: { model_id: modelId, artifact_uri: model.artifact_uri }
|
|
1655
|
+
});
|
|
1656
|
+
}
|
|
1657
|
+
return model;
|
|
1883
1658
|
}
|
|
1884
|
-
return
|
|
1659
|
+
return {
|
|
1660
|
+
root: resolvedRoot,
|
|
1661
|
+
paths,
|
|
1662
|
+
ensure,
|
|
1663
|
+
importSpec: importSpecRecord,
|
|
1664
|
+
async listSpecs() {
|
|
1665
|
+
return listSpecRecords();
|
|
1666
|
+
},
|
|
1667
|
+
async getSpec(id) {
|
|
1668
|
+
const record = findByIdOrPrefix(await listSpecRecords(), id);
|
|
1669
|
+
if (!record) throw new Error(`Spec not found: ${id}`);
|
|
1670
|
+
return { ...record, spec: await readJson2(record.path) };
|
|
1671
|
+
},
|
|
1672
|
+
async startRun({ request, artifactDir }) {
|
|
1673
|
+
await ensure();
|
|
1674
|
+
await importSpecRecord(request.behavior_spec_id, request.spec_snapshot);
|
|
1675
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1676
|
+
const state = {
|
|
1677
|
+
id: request.run_id,
|
|
1678
|
+
behavior_spec_id: request.behavior_spec_id,
|
|
1679
|
+
user_id: request.user_id,
|
|
1680
|
+
run_number: request.run_number,
|
|
1681
|
+
status: "queued",
|
|
1682
|
+
current_stage: "queued",
|
|
1683
|
+
status_message: "Run queued.",
|
|
1684
|
+
artifact_dir: artifactDir,
|
|
1685
|
+
base_model: request.spec_snapshot.base_model,
|
|
1686
|
+
spec_name: request.spec_snapshot.name,
|
|
1687
|
+
created_at: now,
|
|
1688
|
+
updated_at: now,
|
|
1689
|
+
started_at: now
|
|
1690
|
+
};
|
|
1691
|
+
await writeJsonAtomic2(runRequestPath(request.run_id), request);
|
|
1692
|
+
await writeRunState(state);
|
|
1693
|
+
await appendRunEvent(state, { stage: "queued", status: "queued", message: "Run queued." });
|
|
1694
|
+
return state;
|
|
1695
|
+
},
|
|
1696
|
+
async syncRunRequest(request, artifactDir) {
|
|
1697
|
+
const previous = await getRun(request.run_id);
|
|
1698
|
+
if (previous.behavior_spec_id !== request.behavior_spec_id || previous.user_id !== request.user_id || previous.run_number !== request.run_number) {
|
|
1699
|
+
throw new Error(
|
|
1700
|
+
`Run ${request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
|
|
1701
|
+
);
|
|
1702
|
+
}
|
|
1703
|
+
await importSpecRecord(request.behavior_spec_id, request.spec_snapshot);
|
|
1704
|
+
await writeJsonAtomic2(runRequestPath(request.run_id), request);
|
|
1705
|
+
const state = {
|
|
1706
|
+
...previous,
|
|
1707
|
+
artifact_dir: artifactDir ?? previous.artifact_dir,
|
|
1708
|
+
base_model: request.spec_snapshot.base_model,
|
|
1709
|
+
spec_name: request.spec_snapshot.name,
|
|
1710
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1711
|
+
};
|
|
1712
|
+
await writeRunState(state);
|
|
1713
|
+
return state;
|
|
1714
|
+
},
|
|
1715
|
+
async updateRun({ runId, status, stage, message, details }) {
|
|
1716
|
+
const previous = await getRun(runId);
|
|
1717
|
+
if (status !== "cancelled" && await exists2(cancellationPath(previous.id))) {
|
|
1718
|
+
return preserveCancellationRequestState(previous.id);
|
|
1719
|
+
}
|
|
1720
|
+
const active = status !== "stage_completed" && status !== "completed" && status !== "failed" && status !== "cancelled";
|
|
1721
|
+
const successful = status === "stage_completed" || status === "completed";
|
|
1722
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1723
|
+
const state = {
|
|
1724
|
+
...previous,
|
|
1725
|
+
status,
|
|
1726
|
+
current_stage: stage,
|
|
1727
|
+
status_message: message,
|
|
1728
|
+
...active ? { error: void 0, completed_at: void 0 } : successful ? { error: void 0, completed_at: now } : {},
|
|
1729
|
+
updated_at: now
|
|
1730
|
+
};
|
|
1731
|
+
await writeRunState(state);
|
|
1732
|
+
if (status !== "cancelled" && await exists2(cancellationPath(previous.id))) {
|
|
1733
|
+
return preserveCancellationRequestState(previous.id);
|
|
1734
|
+
}
|
|
1735
|
+
await appendRunEvent(state, {
|
|
1736
|
+
stage,
|
|
1737
|
+
status: successful ? "completed" : status === "failed" ? "failed" : "running",
|
|
1738
|
+
message,
|
|
1739
|
+
details
|
|
1740
|
+
});
|
|
1741
|
+
return state;
|
|
1742
|
+
},
|
|
1743
|
+
async completeRun(report, artifactDir, reportPath) {
|
|
1744
|
+
if (await exists2(cancellationPath(report.run_id))) {
|
|
1745
|
+
return preserveCancellationRequestState(report.run_id);
|
|
1746
|
+
}
|
|
1747
|
+
const request = await readJson2(runRequestPath(report.run_id));
|
|
1748
|
+
const completedAt = report.created_at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
1749
|
+
const model = await registerModelRecord({
|
|
1750
|
+
request,
|
|
1751
|
+
training: report.training,
|
|
1752
|
+
artifactDir,
|
|
1753
|
+
createdAt: completedAt
|
|
1754
|
+
});
|
|
1755
|
+
if (!model) {
|
|
1756
|
+
await rm(join2(paths.modelsDir, `local-${report.run_id}`), { recursive: true, force: true });
|
|
1757
|
+
}
|
|
1758
|
+
const previous = await getRun(report.run_id);
|
|
1759
|
+
const state = {
|
|
1760
|
+
...previous,
|
|
1761
|
+
status: "completed",
|
|
1762
|
+
current_stage: "completed",
|
|
1763
|
+
status_message: "Run completed successfully.",
|
|
1764
|
+
report_path: reportPath,
|
|
1765
|
+
model_id: model?.id,
|
|
1766
|
+
error: void 0,
|
|
1767
|
+
completed_at: completedAt,
|
|
1768
|
+
updated_at: completedAt
|
|
1769
|
+
};
|
|
1770
|
+
await writeRunState(state);
|
|
1771
|
+
if (await exists2(cancellationPath(report.run_id))) {
|
|
1772
|
+
return preserveCancellationRequestState(report.run_id);
|
|
1773
|
+
}
|
|
1774
|
+
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
1775
|
+
await appendRunEvent(state, {
|
|
1776
|
+
stage: "completed",
|
|
1777
|
+
status: "completed",
|
|
1778
|
+
message: "Run completed successfully.",
|
|
1779
|
+
details: { report_path: reportPath, ...model ? { model_id: model.id } : {} }
|
|
1780
|
+
});
|
|
1781
|
+
return state;
|
|
1782
|
+
},
|
|
1783
|
+
registerModel: registerModelRecord,
|
|
1784
|
+
async invalidateRunOutputs(runId, options) {
|
|
1785
|
+
const previous = await getRun(runId);
|
|
1786
|
+
if (options.report) await rm(runReportPath(previous.id), { force: true });
|
|
1787
|
+
if (options.model) {
|
|
1788
|
+
await rm(join2(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true });
|
|
1789
|
+
}
|
|
1790
|
+
const state = {
|
|
1791
|
+
...previous,
|
|
1792
|
+
...options.report ? { report_path: void 0 } : {},
|
|
1793
|
+
...options.model ? { model_id: void 0 } : {},
|
|
1794
|
+
error: void 0,
|
|
1795
|
+
completed_at: void 0,
|
|
1796
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1797
|
+
};
|
|
1798
|
+
return writeRunState(state);
|
|
1799
|
+
},
|
|
1800
|
+
async failRun(runId, error) {
|
|
1801
|
+
const previous = await getRun(runId);
|
|
1802
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1803
|
+
if (await exists2(cancellationPath(previous.id))) {
|
|
1804
|
+
return preserveCancellationRequestState(previous.id);
|
|
1805
|
+
}
|
|
1806
|
+
const state = {
|
|
1807
|
+
...previous,
|
|
1808
|
+
status: "failed",
|
|
1809
|
+
current_stage: "failed",
|
|
1810
|
+
status_message: error,
|
|
1811
|
+
error,
|
|
1812
|
+
completed_at: now,
|
|
1813
|
+
updated_at: now
|
|
1814
|
+
};
|
|
1815
|
+
await writeRunState(state);
|
|
1816
|
+
if (await exists2(cancellationPath(previous.id))) {
|
|
1817
|
+
return preserveCancellationRequestState(previous.id);
|
|
1818
|
+
}
|
|
1819
|
+
await appendRunEvent(state, { stage: "failed", status: "failed", message: error });
|
|
1820
|
+
return state;
|
|
1821
|
+
},
|
|
1822
|
+
async cancelRun(runId) {
|
|
1823
|
+
const state = await getRun(runId);
|
|
1824
|
+
if (isTerminalRunState(state) || state.status === "cancelled" && state.current_stage === "cancel_requested") {
|
|
1825
|
+
return;
|
|
1826
|
+
}
|
|
1827
|
+
await writeFile3(cancellationPath(state.id), `${(/* @__PURE__ */ new Date()).toISOString()}
|
|
1828
|
+
`, "utf8");
|
|
1829
|
+
const latest = await getRun(state.id);
|
|
1830
|
+
if (isTerminalRunState(latest)) {
|
|
1831
|
+
if (latest.status !== "cancelled") await rm(cancellationPath(state.id), { force: true });
|
|
1832
|
+
return;
|
|
1833
|
+
}
|
|
1834
|
+
const updated = {
|
|
1835
|
+
...latest,
|
|
1836
|
+
status: "cancelled",
|
|
1837
|
+
current_stage: "cancel_requested",
|
|
1838
|
+
status_message: "Cancellation requested; waiting for the active worker to stop.",
|
|
1839
|
+
error: void 0,
|
|
1840
|
+
completed_at: void 0,
|
|
1841
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1842
|
+
};
|
|
1843
|
+
await writeRunState(updated);
|
|
1844
|
+
await appendRunEvent(updated, { stage: "cancel_requested", status: "cancelled", message: "Cancellation requested." });
|
|
1845
|
+
},
|
|
1846
|
+
async finalizeCancellation(runId) {
|
|
1847
|
+
return finalizeCancellationState(runId);
|
|
1848
|
+
},
|
|
1849
|
+
async isCancellationRequested(runId) {
|
|
1850
|
+
const state = await getRun(runId);
|
|
1851
|
+
return state.status === "cancelled" || await exists2(cancellationPath(state.id));
|
|
1852
|
+
},
|
|
1853
|
+
async listRuns() {
|
|
1854
|
+
return listRunRecords();
|
|
1855
|
+
},
|
|
1856
|
+
getRun,
|
|
1857
|
+
async getRunEvents(id) {
|
|
1858
|
+
const state = await getRun(id);
|
|
1859
|
+
return readJsonl(runEventsPath(state.id));
|
|
1860
|
+
},
|
|
1861
|
+
async getRunReport(id) {
|
|
1862
|
+
const state = await getRun(id);
|
|
1863
|
+
const candidates = [];
|
|
1864
|
+
if (state.report_path) {
|
|
1865
|
+
const reportPath = resolve5(state.report_path);
|
|
1866
|
+
const allowedRoots = [resolvedRoot];
|
|
1867
|
+
if (state.artifact_dir) allowedRoots.push(resolve5(state.artifact_dir));
|
|
1868
|
+
if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) {
|
|
1869
|
+
candidates.push(reportPath);
|
|
1870
|
+
}
|
|
1871
|
+
}
|
|
1872
|
+
candidates.push(runReportPath(state.id));
|
|
1873
|
+
for (const path of candidates) {
|
|
1874
|
+
if (await exists2(path)) return readJson2(path);
|
|
1875
|
+
}
|
|
1876
|
+
throw new Error(`Run has no report yet: ${id}`);
|
|
1877
|
+
},
|
|
1878
|
+
async listModels() {
|
|
1879
|
+
return listModelRecords();
|
|
1880
|
+
},
|
|
1881
|
+
async getModel(id) {
|
|
1882
|
+
const record = findByIdOrPrefix(await listModelRecords(), id);
|
|
1883
|
+
if (!record) throw new Error(`Model not found: ${id}`);
|
|
1884
|
+
return record;
|
|
1885
|
+
}
|
|
1886
|
+
};
|
|
1885
1887
|
}
|
|
1886
1888
|
|
|
1887
1889
|
// src/local-runtime/prefetch.ts
|
|
@@ -1917,6 +1919,7 @@ function buildBundledPythonCommand(entrypoint, args = []) {
|
|
|
1917
1919
|
const commandArgs = [
|
|
1918
1920
|
"run",
|
|
1919
1921
|
"--frozen",
|
|
1922
|
+
"--quiet",
|
|
1920
1923
|
"--project",
|
|
1921
1924
|
bundledProject,
|
|
1922
1925
|
"python",
|
|
@@ -2305,9 +2308,7 @@ async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelC
|
|
|
2305
2308
|
const revision = defaultBaseModelRevision(modelId);
|
|
2306
2309
|
if (!revision) return;
|
|
2307
2310
|
const digests = certifiedFileDigests(modelId);
|
|
2308
|
-
if (!digests)
|
|
2309
|
-
throw new Error(`No certified file digests are registered for pinned model ${modelId}.`);
|
|
2310
|
-
}
|
|
2311
|
+
if (!digests) return;
|
|
2311
2312
|
if (!modelCache) {
|
|
2312
2313
|
throw new Error(`A certified local snapshot for ${modelId} requires paths.modelCache.`);
|
|
2313
2314
|
}
|
|
@@ -4766,10 +4767,6 @@ export {
|
|
|
4766
4767
|
trainingReportSchema,
|
|
4767
4768
|
runReportSchema,
|
|
4768
4769
|
localRunnerConfigSchema,
|
|
4769
|
-
isTerminalRunState,
|
|
4770
|
-
defaultLocalHome,
|
|
4771
|
-
localStorePaths,
|
|
4772
|
-
createLocalStore,
|
|
4773
4770
|
DEFAULT_LOCAL_SPEC_PATH,
|
|
4774
4771
|
resolveLocalRunInputPaths,
|
|
4775
4772
|
initLocalSpecFile,
|
|
@@ -4791,6 +4788,10 @@ export {
|
|
|
4791
4788
|
localRuntimePackageRoot,
|
|
4792
4789
|
buildBundledPythonCommand,
|
|
4793
4790
|
withBundledPythonEnvironment,
|
|
4791
|
+
isTerminalRunState,
|
|
4792
|
+
defaultLocalHome,
|
|
4793
|
+
localStorePaths,
|
|
4794
|
+
createLocalStore,
|
|
4794
4795
|
minimalMachineLearningEnvironment,
|
|
4795
4796
|
withHuggingFaceCacheEnvironment,
|
|
4796
4797
|
withOfflineHuggingFaceCacheEnvironment,
|
|
@@ -4810,4 +4811,4 @@ export {
|
|
|
4810
4811
|
runLocalPipeline,
|
|
4811
4812
|
runLocalFineTune
|
|
4812
4813
|
};
|
|
4813
|
-
//# sourceMappingURL=chunk-
|
|
4814
|
+
//# sourceMappingURL=chunk-YJYCGSHJ.js.map
|