@tuned-tensor/cli 0.25.0 → 0.26.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,14 +1,17 @@
1
1
  #!/usr/bin/env node
2
2
  import {
3
3
  ARTIFACT_WORKFLOW_LOCK_FILE,
4
+ DECISION_MODELS,
4
5
  DEFAULT_ARTIFACT_ROOT,
5
6
  DEFAULT_DECISION_MODEL,
6
7
  DEFAULT_PROJECT_STORE_ROOT,
7
8
  ProcessCancelledError,
9
+ TRAINING_MODELS,
8
10
  assertArtifactManifest,
9
11
  assertUsableModelArtifact,
10
12
  baseModelRevisionSchema,
11
13
  buildBundledPythonCommand,
14
+ buildDecisionPythonCommand,
12
15
  claimRunArtifactDirectory,
13
16
  copyDatasetToTrainingChannel,
14
17
  createProgressRenderer,
@@ -25,6 +28,8 @@ import {
25
28
  isCertifiedTrainingModel,
26
29
  isDecisionSpecFile,
27
30
  isFoundationSpecFile,
31
+ isRepoRelativePath,
32
+ listAssets,
28
33
  localAdapterSpecFileSchema,
29
34
  localBehaviorSpecFileSchema,
30
35
  localDecisionSpecFileSchema,
@@ -51,13 +56,14 @@ import {
51
56
  trainingReportSchema,
52
57
  verifyLocalBaseModel,
53
58
  withBundledPythonEnvironment,
59
+ withDecisionPythonEnvironment,
54
60
  withHuggingFaceCacheEnvironment,
55
61
  withOfflineHuggingFaceCacheEnvironment,
56
62
  writeArtifactManifest,
57
63
  writeFileAtomic,
58
64
  writeJson,
59
65
  writeJsonAtomic
60
- } from "./chunk-UCJCWJ6F.js";
66
+ } from "./chunk-4BHDPRYD.js";
61
67
 
62
68
  // src/local-runtime/dataset.ts
63
69
  import { readFile } from "fs/promises";
@@ -547,10 +553,10 @@ function runRequestFromLocalSpec(spec, options = {}) {
547
553
  }
548
554
  function deterministicBehaviorSpecId(spec) {
549
555
  const namespace = Buffer.from("9f783f01340457bea061942004259253", "hex");
550
- const digest = createHash("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
551
- digest[6] = digest[6] & 15 | 80;
552
- digest[8] = digest[8] & 63 | 128;
553
- const hex = digest.toString("hex");
556
+ const digest2 = createHash("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
557
+ digest2[6] = digest2[6] & 15 | 80;
558
+ digest2[8] = digest2[8] & 63 | 128;
559
+ const hex = digest2.toString("hex");
554
560
  return [
555
561
  hex.slice(0, 8),
556
562
  hex.slice(8, 12),
@@ -840,14 +846,14 @@ function createExecutionPlan(pipeline, selection = {}) {
840
846
  });
841
847
  return { version: 1, ...normalized.name ? { name: normalized.name } : {}, steps: resolved };
842
848
  }
843
- function assertFoundationSettingsMatch(hp, plan) {
849
+ function assertFoundationSettingsMatch(hp, plan2) {
844
850
  const settings = {
845
851
  tokenize: { vocabSize: hp.vocab_size, maxChars: hp.max_chars },
846
852
  pretrain: { depth: hp.depth, steps: hp.pretrain_steps, batchSize: hp.batch_size, sequenceLength: hp.sequence_length, nprocPerNode: hp.nproc_per_node },
847
853
  finetune: { steps: hp.finetune_steps, batchSize: hp.batch_size },
848
854
  rl: { steps: hp.rl_steps }
849
855
  };
850
- for (const step of plan.steps) {
856
+ for (const step of plan2.steps) {
851
857
  const fields = "with" in step ? step.with : void 0;
852
858
  for (const [key, expected] of Object.entries(settings[step.uses] ?? {})) {
853
859
  if (fields?.[key] !== void 0 && fields[key] !== expected) {
@@ -858,25 +864,949 @@ function assertFoundationSettingsMatch(hp, plan) {
858
864
  }
859
865
  }
860
866
 
861
- // src/local-runtime/audit.ts
867
+ // src/local-runtime/dataset-split.ts
868
+ import { createHash as createHash3, randomUUID as randomUUID4 } from "crypto";
869
+ import { constants as constants4 } from "fs";
870
+ import { lstat as lstat3, mkdir as mkdir3, open as open4, rename as rename2, rm, rmdir, writeFile as writeFile3 } from "fs/promises";
871
+ import { dirname as dirname3, isAbsolute as isAbsolute4, join as join4, relative as relative4, resolve as resolve2, sep as sep4 } from "path";
872
+ import { z } from "zod";
873
+
874
+ // src/local-spec-workspace.ts
862
875
  import { createHash as createHash2, randomUUID as randomUUID2 } from "crypto";
876
+ import { constants } from "fs";
877
+ import { lstat, open, realpath, writeFile as writeFile2 } from "fs/promises";
878
+ import { join } from "path";
879
+ import { Type } from "typebox";
880
+ import { Value } from "typebox/value";
881
+ var LocalBaseModel = Type.String({
882
+ minLength: 3,
883
+ maxLength: 193,
884
+ pattern: "^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$",
885
+ description: `Hugging Face repo id. Certified adapter models: ${TRAINING_MODELS.map((model) => model.id).join(", ")}. Decision models: ${DECISION_MODELS.map((model) => model.id).join(", ")}. Other adapter repos require hyperparameters.base_model_revision (40-character commit SHA).`
886
+ });
887
+ var NonEmptyText = Type.String({ minLength: 1, maxLength: 1e3 });
888
+ var LocalProjectSpecSchema = Type.Object({
889
+ name: Type.String({ minLength: 1, maxLength: 255 }),
890
+ description: Type.Optional(Type.String({ maxLength: 5e3 })),
891
+ base_model: Type.Optional(LocalBaseModel),
892
+ engine: Type.Optional(Type.Union([Type.Literal("adapter"), Type.Literal("foundation"), Type.Literal("decision")])),
893
+ decision: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
894
+ hyperparameters: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
895
+ foundation: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
896
+ dataset_prebuilt: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
897
+ evaluation: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
898
+ runtime: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
899
+ pipeline: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
900
+ system_prompt: Type.String({ minLength: 1, maxLength: 1e4 }),
901
+ guidelines: Type.Array(NonEmptyText, { minItems: 1, maxItems: 50 }),
902
+ constraints: Type.Optional(Type.Array(NonEmptyText, { maxItems: 50 })),
903
+ examples: Type.Array(Type.Object({
904
+ input: Type.String({ minLength: 1, maxLength: 1e5 }),
905
+ output: Type.String({ minLength: 1, maxLength: 1e5 })
906
+ }, { additionalProperties: false }), { minItems: 2, maxItems: 500 })
907
+ }, { additionalProperties: false });
908
+ var SPEC_FILE = "tunedtensor.json";
909
+ var SPEC_DISPLAY_PATH = `./${SPEC_FILE}`;
910
+ var LocalSpecMutationError = class extends Error {
911
+ constructor(message, outcome, cause) {
912
+ super(message, { cause });
913
+ this.outcome = outcome;
914
+ this.name = "LocalSpecMutationError";
915
+ }
916
+ };
917
+ function isKnownLocalSpecFailure(error) {
918
+ return error instanceof LocalSpecMutationError && error.outcome === "not_applied";
919
+ }
920
+ var NODE_FILE_OPERATIONS = { writeFile: writeFile2 };
921
+ function errorCode(error) {
922
+ return typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
923
+ }
924
+ function validateInputs(spec) {
925
+ if (!Value.Check(LocalProjectSpecSchema, spec)) {
926
+ throw new Error("Local spec content does not match the canonical tunedtensor.json schema.");
927
+ }
928
+ pipelineForRunInput(parseLocalRunInput(spec, SPEC_FILE));
929
+ }
930
+ async function canonicalWorkspace(workspaceRoot) {
931
+ const rootInfo = await lstat(workspaceRoot);
932
+ if (rootInfo.isSymbolicLink() || !rootInfo.isDirectory()) {
933
+ throw new Error("The local agent workspace must be a real directory, not a symlink.");
934
+ }
935
+ return await realpath(workspaceRoot);
936
+ }
937
+ async function fingerprintWorkspace(canonicalRoot) {
938
+ return fingerprintWorkspaceInfo(await lstat(canonicalRoot));
939
+ }
940
+ function fingerprintWorkspaceInfo(info) {
941
+ const identity = `${info.dev}:${info.ino}:${info.birthtimeMs}`;
942
+ return createHash2("sha256").update(identity).digest("hex");
943
+ }
944
+ async function prepareLocalSpecProject(workspaceRoot, spec, expectedWorkspaceFingerprint) {
945
+ const canonicalRoot = await canonicalWorkspace(workspaceRoot);
946
+ const workspaceFingerprint = await fingerprintWorkspace(canonicalRoot);
947
+ if (expectedWorkspaceFingerprint !== void 0 && workspaceFingerprint !== expectedWorkspaceFingerprint) {
948
+ throw new Error(
949
+ "The local workspace changed after this action was prepared; return to the original workspace and prepare it again."
950
+ );
951
+ }
952
+ validateInputs(spec);
953
+ try {
954
+ await lstat(join(canonicalRoot, SPEC_FILE));
955
+ } catch (error) {
956
+ if (errorCode(error) === "ENOENT") {
957
+ return { specPath: SPEC_DISPLAY_PATH, workspaceFingerprint };
958
+ }
959
+ throw error;
960
+ }
961
+ throw new Error(
962
+ `Refusing to overwrite existing ${SPEC_DISPLAY_PATH}; edit it with prepare_update_local_spec instead.`
963
+ );
964
+ }
965
+ async function createLocalSpecProject(workspaceRoot, spec, expectedWorkspaceFingerprint, operationOverrides = {}) {
966
+ let prepared;
967
+ let canonicalRoot;
968
+ try {
969
+ prepared = await prepareLocalSpecProject(workspaceRoot, spec, expectedWorkspaceFingerprint);
970
+ canonicalRoot = await canonicalWorkspace(workspaceRoot);
971
+ } catch (error) {
972
+ const detail = error instanceof Error ? error.message : String(error);
973
+ throw new LocalSpecMutationError(detail, "not_applied", error);
974
+ }
975
+ if (process.platform !== "linux") {
976
+ throw new LocalSpecMutationError(
977
+ "Secure local spec creation currently requires Linux filesystem handle support.",
978
+ "not_applied"
979
+ );
980
+ }
981
+ let workspaceHandle;
982
+ try {
983
+ workspaceHandle = await open(
984
+ canonicalRoot,
985
+ constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW
986
+ );
987
+ if (fingerprintWorkspaceInfo(await workspaceHandle.stat()) !== expectedWorkspaceFingerprint) {
988
+ throw new LocalSpecMutationError(
989
+ "The local workspace changed after this action was prepared; return to the original workspace and prepare it again.",
990
+ "not_applied"
991
+ );
992
+ }
993
+ } catch (error) {
994
+ if (workspaceHandle) {
995
+ try {
996
+ await workspaceHandle.close();
997
+ } catch {
998
+ }
999
+ }
1000
+ if (error instanceof LocalSpecMutationError) throw error;
1001
+ const detail = error instanceof Error ? error.message : String(error);
1002
+ throw new LocalSpecMutationError(detail, "not_applied", error);
1003
+ }
1004
+ const operations = { ...NODE_FILE_OPERATIONS, ...operationOverrides };
1005
+ const stableSpecPath = join(`/proc/self/fd/${workspaceHandle.fd}`, SPEC_FILE);
1006
+ try {
1007
+ await operations.writeFile(stableSpecPath, `${JSON.stringify({ id: randomUUID2(), ...spec }, null, 2)}
1008
+ `, {
1009
+ encoding: "utf8",
1010
+ flag: "wx",
1011
+ mode: 384
1012
+ });
1013
+ const finalWorkspaceInfo = await lstat(canonicalRoot);
1014
+ if (finalWorkspaceInfo.isSymbolicLink() || !finalWorkspaceInfo.isDirectory() || fingerprintWorkspaceInfo(finalWorkspaceInfo) !== expectedWorkspaceFingerprint) {
1015
+ throw new Error("The local workspace changed while the spec was being written.");
1016
+ }
1017
+ } catch (error) {
1018
+ try {
1019
+ await workspaceHandle.close();
1020
+ } catch {
1021
+ }
1022
+ const detail = error instanceof Error ? error.message : String(error);
1023
+ if (errorCode(error) === "EEXIST") {
1024
+ throw new LocalSpecMutationError(`Refusing to overwrite existing ${SPEC_DISPLAY_PATH}.`, "not_applied", error);
1025
+ }
1026
+ throw new LocalSpecMutationError(
1027
+ `Local spec writing did not complete conclusively for ${SPEC_DISPLAY_PATH}; inspect the workspace before retrying. ${detail}`,
1028
+ "unknown",
1029
+ error
1030
+ );
1031
+ }
1032
+ try {
1033
+ await workspaceHandle.close();
1034
+ } catch {
1035
+ }
1036
+ return { created: true, path: prepared.specPath };
1037
+ }
1038
+
1039
+ // src/spec-workspace.ts
1040
+ import { randomUUID as randomUUID3 } from "crypto";
1041
+ import { constants as constants2 } from "fs";
1042
+ import { lstat as lstat2, mkdir as mkdir2, open as open2, readdir, rename, unlink } from "fs/promises";
1043
+ import { isAbsolute as isAbsolute2, join as join2, relative as relative2, sep as sep2 } from "path";
1044
+ var MAX_BYTES = 2e5;
1045
+ var EDIT_KEYS = /* @__PURE__ */ new Set([
1046
+ "name",
1047
+ "description",
1048
+ "system_prompt",
1049
+ "guidelines",
1050
+ "constraints",
1051
+ "examples",
1052
+ "base_model",
1053
+ "hyperparameters",
1054
+ "foundation",
1055
+ "decision",
1056
+ "dataset_prebuilt",
1057
+ "runtime",
1058
+ "evaluation",
1059
+ "pipeline",
1060
+ "add_examples"
1061
+ ]);
1062
+ function isObject(value) {
1063
+ return value !== null && typeof value === "object" && !Array.isArray(value);
1064
+ }
1065
+ function validateDocument(document, path) {
1066
+ try {
1067
+ const input = parseLocalRunInput(document, path);
1068
+ const result = validateBehaviorSpec(input);
1069
+ if (result.valid) pipelineForRunInput(input);
1070
+ return result;
1071
+ } catch (error) {
1072
+ return { valid: false, errors: [error.message], warnings: [] };
1073
+ }
1074
+ }
1075
+ async function readBounded(path, limit = MAX_BYTES) {
1076
+ const handle = await open2(path, constants2.O_RDONLY | constants2.O_NOFOLLOW | constants2.O_NONBLOCK);
1077
+ try {
1078
+ const info = await handle.stat();
1079
+ if (!info.isFile() || info.size > limit) throw new Error(`Spec/revision must be a regular file of at most ${limit} bytes.`);
1080
+ const buffer = Buffer.alloc(limit + 1);
1081
+ let size = 0;
1082
+ while (size < buffer.length) {
1083
+ const result = await handle.read(buffer, size, buffer.length - size, null);
1084
+ if (!result.bytesRead) break;
1085
+ size += result.bytesRead;
1086
+ }
1087
+ if (size > limit) throw new Error(`Spec/revision exceeds ${limit} bytes.`);
1088
+ return buffer.subarray(0, size).toString("utf8");
1089
+ } finally {
1090
+ await handle.close();
1091
+ }
1092
+ }
1093
+ async function specLocation(workspaceRoot, specPath) {
1094
+ const root = await canonicalWorkspace(workspaceRoot);
1095
+ const parts = specPath.replace(/^\.\//, "").split(/[\\/]/);
1096
+ if (isAbsolute2(specPath) || parts.some((part) => !part || part === "." || part === "..") || parts.at(-1) !== "tunedtensor.json") {
1097
+ throw new Error("spec_path must name tunedtensor.json inside the current workspace (no traversal or symlinks).");
1098
+ }
1099
+ let parent = root;
1100
+ for (const part of parts.slice(0, -1)) {
1101
+ parent = join2(parent, part);
1102
+ const info = await lstat2(parent);
1103
+ if (info.isSymbolicLink() || !info.isDirectory()) throw new Error("Spec directories must be real directories, not symlinks.");
1104
+ }
1105
+ return {
1106
+ root,
1107
+ path: join2(parent, "tunedtensor.json"),
1108
+ parent,
1109
+ displayPath: `./${parts.join("/")}`,
1110
+ workspaceFingerprint: await fingerprintWorkspace(root),
1111
+ parentFingerprint: await fingerprintWorkspace(parent)
1112
+ };
1113
+ }
1114
+ async function assertCurrentLocation(location2) {
1115
+ const root = await canonicalWorkspace(location2.root);
1116
+ if (await fingerprintWorkspace(root) !== location2.workspaceFingerprint || await fingerprintWorkspace(location2.parent) !== location2.parentFingerprint) {
1117
+ throw new Error("The spec workspace changed during editing; inspect the revision journal before retrying.");
1118
+ }
1119
+ }
1120
+ async function readSpecSource(location2) {
1121
+ if (process.platform !== "linux") return await readBounded(location2.path);
1122
+ const handles = [];
1123
+ try {
1124
+ let directory = await openDirectory(location2.root);
1125
+ handles.push(directory);
1126
+ if (await fingerprintWorkspace(`/proc/self/fd/${directory.fd}/.`) !== location2.workspaceFingerprint) throw new Error("Workspace changed during spec inspection.");
1127
+ for (const part of relative2(location2.root, location2.parent).split(sep2).filter(Boolean)) {
1128
+ directory = await openDirectory(`/proc/self/fd/${directory.fd}/${part}`);
1129
+ handles.push(directory);
1130
+ }
1131
+ if (await fingerprintWorkspace(`/proc/self/fd/${directory.fd}/.`) !== location2.parentFingerprint) throw new Error("Spec directory changed during inspection.");
1132
+ return await readBounded(`/proc/self/fd/${directory.fd}/tunedtensor.json`);
1133
+ } finally {
1134
+ for (const handle of handles.reverse()) await handle.close();
1135
+ }
1136
+ }
1137
+ async function inspectLocalSpec(workspaceRoot, specPath = "tunedtensor.json", options = {}) {
1138
+ const location2 = await specLocation(workspaceRoot, specPath);
1139
+ const source = await readSpecSource(location2).catch((error) => {
1140
+ if (options.allowMissing && error.code === "ENOENT") return "";
1141
+ throw error;
1142
+ });
1143
+ let document;
1144
+ let validation;
1145
+ try {
1146
+ document = JSON.parse(source);
1147
+ validation = validateDocument(document, location2.path);
1148
+ } catch (error) {
1149
+ validation = { valid: false, errors: [`Invalid JSON: ${error.message}`], warnings: [] };
1150
+ }
1151
+ return { ...location2, source, sha256: specHash(source), document, validation };
1152
+ }
1153
+ function arrayDiff(before, after) {
1154
+ const added = after.map((item) => JSON.stringify(item));
1155
+ const removed = [];
1156
+ for (const item of before.map((value) => JSON.stringify(value))) {
1157
+ const index = added.indexOf(item);
1158
+ if (index === -1) removed.push(`- ${item}`);
1159
+ else added.splice(index, 1);
1160
+ }
1161
+ const lines = [...removed, ...added.map((item) => `+ ${item}`)];
1162
+ return lines.length ? lines : [" (order changed)"];
1163
+ }
1164
+ function derivedSpecId(document, path) {
1165
+ if (document.id !== void 0) return void 0;
1166
+ try {
1167
+ const input = parseLocalRunInput(document, path);
1168
+ return input.kind === "spec" ? input.request.behavior_spec_id : void 0;
1169
+ } catch {
1170
+ return void 0;
1171
+ }
1172
+ }
1173
+ function specDiff(before, after) {
1174
+ const left = isObject(before) ? before : {};
1175
+ const right = isObject(after) ? after : {};
1176
+ const keys = [.../* @__PURE__ */ new Set([...Object.keys(left), ...Object.keys(right)])].sort();
1177
+ const lines = [];
1178
+ for (const key of keys) {
1179
+ if (JSON.stringify(left[key]) === JSON.stringify(right[key])) continue;
1180
+ if (Array.isArray(left[key]) && Array.isArray(right[key])) {
1181
+ lines.push(`@@ ${key} @@`, ...arrayDiff(left[key], right[key]));
1182
+ continue;
1183
+ }
1184
+ lines.push(
1185
+ `@@ ${key} @@`,
1186
+ ...JSON.stringify(left[key] ?? null, null, 2).split("\n").map((line) => `- ${line}`),
1187
+ ...JSON.stringify(right[key] ?? null, null, 2).split("\n").map((line) => `+ ${line}`)
1188
+ );
1189
+ }
1190
+ return lines.join("\n") || "No changes.";
1191
+ }
1192
+ async function prepareSpecUpdate(workspaceRoot, specPath, expectedSha256, changes) {
1193
+ const current = await inspectLocalSpec(workspaceRoot, specPath);
1194
+ if (current.sha256 !== expectedSha256) throw new Error("The behavior spec changed; inspect it and prepare the edit again.");
1195
+ if (!isObject(current.document)) throw new Error("Repair the spec JSON before preparing an edit.");
1196
+ if (!isObject(changes) || !Object.keys(changes).length || Object.keys(changes).some((key) => !EDIT_KEYS.has(key))) {
1197
+ throw new Error("Provide supported spec field changes; identity and engine cannot be changed by an edit.");
1198
+ }
1199
+ const { add_examples: added, ...fields } = changes;
1200
+ if (added !== void 0) {
1201
+ if (!Array.isArray(added) || "examples" in fields) throw new Error("Use add_examples to append examples, or examples to replace them, not both.");
1202
+ const existing = Array.isArray(current.document.examples) ? current.document.examples : [];
1203
+ fields.examples = [...existing, ...added];
1204
+ }
1205
+ const id = derivedSpecId(current.document, current.path);
1206
+ const next = { ...id ? { id } : {}, ...current.document, ...fields };
1207
+ function mergeSettings(before, patch) {
1208
+ const result = { ...before };
1209
+ for (const [key, value] of Object.entries(patch)) {
1210
+ if (value === null) delete result[key];
1211
+ else result[key] = isObject(value) && isObject(result[key]) ? mergeSettings(result[key], value) : value;
1212
+ }
1213
+ return result;
1214
+ }
1215
+ for (const key of ["runtime", "evaluation", "pipeline"]) {
1216
+ if (fields[key] === null) delete next[key];
1217
+ else if (key !== "pipeline" && isObject(fields[key])) next[key] = mergeSettings(isObject(current.document[key]) ? current.document[key] : {}, fields[key]);
1218
+ }
1219
+ for (const key of ["hyperparameters", "foundation"]) {
1220
+ if (isObject(fields[key])) {
1221
+ const previous = isObject(current.document[key]) ? current.document[key] : {};
1222
+ next[key] = { ...previous, ...fields[key] };
1223
+ }
1224
+ }
1225
+ const validation = validateDocument(next, current.path);
1226
+ if (!validation.valid) throw new Error(`Invalid behavior spec:
1227
+ ${validation.errors.join("\n")}`);
1228
+ const source = `${JSON.stringify(next, null, 2)}
1229
+ `;
1230
+ if (Buffer.byteLength(source) > MAX_BYTES) throw new Error("Updated spec is too large.");
1231
+ const diff = specDiff(current.document, next);
1232
+ if (diff === "No changes.") throw new Error("The proposed edit makes no changes.");
1233
+ const update = {
1234
+ spec_path: current.displayPath,
1235
+ expected_sha256: current.sha256,
1236
+ after_sha256: specHash(source),
1237
+ workspace_fingerprint: current.workspaceFingerprint,
1238
+ parent_fingerprint: current.parentFingerprint,
1239
+ changes
1240
+ };
1241
+ return { current, next, source, diff, validation, update };
1242
+ }
1243
+ var SpecWorkspaceMismatchError = class extends Error {
1244
+ };
1245
+ async function validateSpecUpdate(workspaceRoot, update) {
1246
+ const root = await canonicalWorkspace(workspaceRoot);
1247
+ if (await fingerprintWorkspace(root) !== update.workspace_fingerprint) {
1248
+ throw new SpecWorkspaceMismatchError("The local workspace changed; return to the original workspace before approving.");
1249
+ }
1250
+ const prepared = await prepareSpecUpdate(workspaceRoot, update.spec_path, update.expected_sha256, update.changes);
1251
+ if (prepared.current.workspaceFingerprint !== update.workspace_fingerprint || prepared.current.parentFingerprint !== update.parent_fingerprint) {
1252
+ throw new Error("The spec workspace changed; return to the original workspace and prepare the edit again.");
1253
+ }
1254
+ if (prepared.update.after_sha256 !== update.after_sha256) {
1255
+ throw new Error("The proposed spec edit changed after review; prepare it again.");
1256
+ }
1257
+ return prepared;
1258
+ }
1259
+ async function openDirectory(path) {
1260
+ return await open2(path, constants2.O_RDONLY | constants2.O_DIRECTORY | constants2.O_NOFOLLOW);
1261
+ }
1262
+ async function privateChild(parent, name) {
1263
+ const path = `/proc/self/fd/${parent.fd}/${name}`;
1264
+ await mkdir2(path, { mode: 448 }).catch((error) => {
1265
+ if (error.code !== "EEXIST") throw error;
1266
+ });
1267
+ return await openDirectory(path);
1268
+ }
1269
+ async function atomicText(parent, name, source) {
1270
+ const base = `/proc/self/fd/${parent.fd}`;
1271
+ const temporary = join2(base, `.tt-${randomUUID3()}.tmp`);
1272
+ try {
1273
+ const file = await open2(temporary, "wx", 384);
1274
+ try {
1275
+ await file.writeFile(source, "utf8");
1276
+ await file.sync();
1277
+ } finally {
1278
+ await file.close();
1279
+ }
1280
+ await rename(temporary, join2(base, name));
1281
+ await parent.sync();
1282
+ } finally {
1283
+ await unlink(temporary).catch(() => {
1284
+ });
1285
+ }
1286
+ }
1287
+ async function applySpecUpdate(workspaceRoot, update) {
1288
+ let committing = false;
1289
+ let lock;
1290
+ const handles = [];
1291
+ let lockPath;
1292
+ try {
1293
+ if (process.platform !== "linux") throw new Error("Secure local spec editing currently requires Linux filesystem handle support.");
1294
+ const prepared = await validateSpecUpdate(workspaceRoot, update);
1295
+ let parent = await openDirectory(prepared.current.root);
1296
+ handles.push(parent);
1297
+ for (const part of relative2(prepared.current.root, prepared.current.parent).split(sep2).filter(Boolean)) {
1298
+ parent = await openDirectory(`/proc/self/fd/${parent.fd}/${part}`);
1299
+ handles.push(parent);
1300
+ }
1301
+ const stable = `/proc/self/fd/${parent.fd}`;
1302
+ if (await fingerprintWorkspace(`${stable}/.`) !== update.parent_fingerprint) throw new Error("Spec directory changed before editing.");
1303
+ lockPath = `${stable}/.tunedtensor-spec.lock`;
1304
+ lock = await open2(lockPath, "wx", 384);
1305
+ const before = await readBounded(`${stable}/tunedtensor.json`);
1306
+ if (specHash(before) !== update.expected_sha256) throw new Error("The behavior spec changed; inspect it and prepare the edit again.");
1307
+ const state = await privateChild(parent, ".tuned-tensor");
1308
+ handles.push(state);
1309
+ const history = await privateChild(state, "spec-history");
1310
+ handles.push(history);
1311
+ const id = `${Date.now()}-${randomUUID3()}`;
1312
+ const revision = {
1313
+ id,
1314
+ status: "prepared",
1315
+ at: (/* @__PURE__ */ new Date()).toISOString(),
1316
+ before_sha256: update.expected_sha256,
1317
+ after_sha256: update.after_sha256,
1318
+ before: prepared.current.document,
1319
+ after: prepared.next
1320
+ };
1321
+ await atomicText(history, `${id}.json`, JSON.stringify(revision));
1322
+ if (specHash(await readBounded(`${stable}/tunedtensor.json`)) !== update.expected_sha256) throw new Error("The behavior spec changed before commit; prepare the edit again.");
1323
+ await assertCurrentLocation(prepared.current);
1324
+ committing = true;
1325
+ await atomicText(parent, "tunedtensor.json", prepared.source);
1326
+ await assertCurrentLocation(prepared.current);
1327
+ await atomicText(history, `${id}.json`, JSON.stringify({ ...revision, status: "applied" }));
1328
+ return { updated: true, path: update.spec_path, revision: id, sha256: revision.after_sha256, diff: prepared.diff };
1329
+ } catch (error) {
1330
+ throw new LocalSpecMutationError(error.message, committing ? "unknown" : "not_applied", error);
1331
+ } finally {
1332
+ if (lock) {
1333
+ await lock.close();
1334
+ await unlink(lockPath).catch(() => {
1335
+ });
1336
+ }
1337
+ for (const handle of handles.reverse()) await handle.close();
1338
+ }
1339
+ }
1340
+ async function readSpecHistory(workspaceRoot, specPath = "tunedtensor.json") {
1341
+ const location2 = await specLocation(workspaceRoot, specPath);
1342
+ let path = location2.parent;
1343
+ for (const part of [".tuned-tensor", "spec-history"]) {
1344
+ path = join2(path, part);
1345
+ try {
1346
+ const info = await lstat2(path);
1347
+ if (!info.isDirectory() || info.isSymbolicLink()) {
1348
+ throw new Error("Spec history must use real directories.");
1349
+ }
1350
+ } catch (error) {
1351
+ if (error.code === "ENOENT") return [];
1352
+ throw error;
1353
+ }
1354
+ }
1355
+ const names = (await readdir(path)).filter((name) => /^\d+-[a-f0-9-]+\.json$/.test(name)).sort().reverse().slice(0, 50);
1356
+ return await Promise.all(names.map(async (name) => JSON.parse(await readBounded(join2(path, name), MAX_BYTES * 3))));
1357
+ }
1358
+
1359
+ // src/agent-local-datasets.ts
1360
+ import { constants as constants3 } from "fs";
1361
+ import { open as open3, realpath as realpath2, stat as stat2 } from "fs/promises";
1362
+ import { basename, dirname as dirname2, isAbsolute as isAbsolute3, join as join3, relative as relative3, sep as sep3 } from "path";
1363
+ import { parquetMetadataAsync, parquetReadObjects, parquetSchema } from "hyparquet";
1364
+ import { compressors } from "hyparquet-compressors";
1365
+ var MAX_READ_BYTES = 16 * 1024 * 1024;
1366
+ var SAMPLE_ROWS = 2;
1367
+ var SUPPORTED_FILE = /\.(jsonl|parquet)$/i;
1368
+ async function datasets() {
1369
+ return (await listAssets()).filter((asset) => asset.kind === "dataset");
1370
+ }
1371
+ async function cachedPath(asset, file) {
1372
+ if (!isRepoRelativePath(file) || file.includes("\\") || !Object.hasOwn(asset.files, file)) {
1373
+ throw new Error("Choose an exact recorded file from list_local_datasets; arbitrary paths are not supported.");
1374
+ }
1375
+ const snapshot = asset.snapshot_path;
1376
+ const repoRoot = dirname2(dirname2(snapshot));
1377
+ if (!/^[a-f0-9]{40}$/.test(asset.revision) || basename(snapshot) !== asset.revision || basename(dirname2(snapshot)) !== "snapshots" || basename(repoRoot) !== `datasets--${asset.repo.replaceAll("/", "--")}`) {
1378
+ throw new Error("The dataset manifest has an invalid cache location.");
1379
+ }
1380
+ const root = await realpath2(repoRoot);
1381
+ const target = await realpath2(join3(snapshot, file));
1382
+ const child = relative3(root, target);
1383
+ if (isAbsolute3(child) || child.startsWith(`..${sep3}`) || child === ".." || !child) {
1384
+ throw new Error("The recorded dataset file points outside its cache directory.");
1385
+ }
1386
+ if (!(await stat2(target)).isFile()) throw new Error("The cached dataset must be a regular file.");
1387
+ return target;
1388
+ }
1389
+ async function resolveCachedDatasetFile(source) {
1390
+ const match = /^hf:\/\/datasets\/([^/]+\/[^/@]+)@([a-f0-9]{40})\/(.+)$/.exec(source);
1391
+ if (!match) throw new Error("Use the pinned split_source returned by inspect_local_dataset.");
1392
+ const [, repo, revision, file] = match;
1393
+ const asset = (await datasets()).find((item) => item.repo === repo && item.revision === revision);
1394
+ if (!asset) throw new Error("No TT-downloaded dataset matches that repository and revision.");
1395
+ const path = await cachedPath(asset, file);
1396
+ const expectedDigest = asset.files[file];
1397
+ if (typeof expectedDigest !== "string" || !/^(sha256:[a-f0-9]{64}|git-sha1:[a-f0-9]{40})$/.test(expectedDigest)) {
1398
+ throw new Error("The cached dataset has no valid recorded checksum; download it again.");
1399
+ }
1400
+ return { path, expectedDigest };
1401
+ }
1402
+ async function listLocalDatasets(page = 1, perPage = 20) {
1403
+ const all = await datasets();
1404
+ const selected = all.slice((page - 1) * perPage, page * perPage);
1405
+ return {
1406
+ total: all.length,
1407
+ page,
1408
+ per_page: perPage,
1409
+ datasets: await Promise.all(selected.map(async (asset) => {
1410
+ const files = Object.keys(asset.files).filter((file) => SUPPORTED_FILE.test(file)).sort();
1411
+ return {
1412
+ repo: asset.repo,
1413
+ revision: asset.revision,
1414
+ fetched_at: asset.fetched_at,
1415
+ size_bytes: asset.size_bytes,
1416
+ file_count: files.length,
1417
+ files: await Promise.all(files.slice(0, 100).map(async (file) => ({
1418
+ file,
1419
+ available: await cachedPath(asset, file).then(() => true, () => false)
1420
+ }))),
1421
+ files_truncated: files.length > 100
1422
+ };
1423
+ })),
1424
+ note: "These are TT-recorded local downloads. Availability is checked, checksums are not rehashed. Inspect the selected files with inspect_local_dataset before mapping columns. Adapter fine-tuning needs training plus a distinct validation/test file; never reuse the training file for evaluation."
1425
+ };
1426
+ }
1427
+ function preview(value, depth = 0) {
1428
+ if (typeof value === "string") return value.length > 600 ? `${value.slice(0, 600)}\u2026 [truncated]` : value;
1429
+ if (typeof value === "bigint") return String(value);
1430
+ if (value === null || typeof value !== "object") return value;
1431
+ if (depth >= 4) return "[nested value omitted]";
1432
+ if (Array.isArray(value)) return value.slice(0, 4).map((item) => preview(item, depth + 1));
1433
+ return Object.fromEntries(Object.entries(value).slice(0, 12).map(([key, item]) => [key, preview(item, depth + 1)]));
1434
+ }
1435
+ async function inspectLocalDataset(repo, revision, fileName) {
1436
+ const asset = (await datasets()).find((item) => item.repo === repo && item.revision === revision);
1437
+ if (!asset) throw new Error("No TT-downloaded dataset matches that repository and revision. Use list_local_datasets first.");
1438
+ if (!SUPPORTED_FILE.test(fileName)) throw new Error("Only JSONL and Parquet dataset files can be inspected.");
1439
+ const path = await cachedPath(asset, fileName);
1440
+ const handle = await open3(path, constants3.O_RDONLY | constants3.O_NOFOLLOW | constants3.O_NONBLOCK);
1441
+ try {
1442
+ const info = await handle.stat();
1443
+ if (!info.isFile()) throw new Error("The cached dataset must be a regular file.");
1444
+ let bytesRead = 0;
1445
+ const file = {
1446
+ byteLength: info.size,
1447
+ slice: async (start, end = info.size) => {
1448
+ if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end > info.size) {
1449
+ throw new Error("Invalid dataset read range.");
1450
+ }
1451
+ bytesRead += end - start;
1452
+ if (bytesRead > MAX_READ_BYTES) throw new Error("Dataset inspection exceeds the 16 MiB read limit.");
1453
+ const buffer = Buffer.alloc(end - start);
1454
+ let offset = 0;
1455
+ while (offset < buffer.length) {
1456
+ const read = await handle.read(buffer, offset, buffer.length - offset, start + offset);
1457
+ if (!read.bytesRead) throw new Error("Dataset changed during inspection; inspect it again.");
1458
+ offset += read.bytesRead;
1459
+ }
1460
+ return Uint8Array.from(buffer).buffer;
1461
+ }
1462
+ };
1463
+ let columns;
1464
+ let rowCount = null;
1465
+ let rows = [];
1466
+ let sampleError;
1467
+ if (fileName.toLowerCase().endsWith(".parquet")) {
1468
+ const metadata = await parquetMetadataAsync(file);
1469
+ columns = parquetSchema(metadata).children.map((child) => child.element.name);
1470
+ rowCount = String(metadata.num_rows);
1471
+ try {
1472
+ rows = await parquetReadObjects({ file, metadata, compressors, columns: columns.slice(0, 12), rowStart: 0, rowEnd: Math.min(SAMPLE_ROWS, Number(metadata.num_rows)) });
1473
+ } catch (error) {
1474
+ sampleError = error.message;
1475
+ }
1476
+ } else {
1477
+ const length = Math.min(info.size, 256 * 1024);
1478
+ const text = Buffer.from(await file.slice(0, length)).toString("utf8");
1479
+ const lines = (length < info.size ? text.slice(0, text.lastIndexOf("\n") + 1) : text).split("\n").filter((line) => line.trim());
1480
+ rows = lines.slice(0, SAMPLE_ROWS).map((line) => {
1481
+ const row = JSON.parse(line);
1482
+ if (!row || typeof row !== "object" || Array.isArray(row)) throw new Error("Expected JSONL objects in the cached dataset.");
1483
+ return row;
1484
+ });
1485
+ if (!rows.length) throw new Error("No complete JSONL records found in the first 256 KiB.");
1486
+ columns = [...new Set(rows.flatMap((row) => Object.keys(row)))];
1487
+ }
1488
+ return {
1489
+ repo: asset.repo,
1490
+ revision: asset.revision,
1491
+ file: fileName,
1492
+ size_bytes: info.size,
1493
+ columns: columns.slice(0, 100),
1494
+ columns_truncated: columns.length > 100,
1495
+ schema_source: fileName.toLowerCase().endsWith(".parquet") ? "parquet_metadata" : "sampled_jsonl_records",
1496
+ row_count: rowCount,
1497
+ sample: rows.map((row) => preview(row)),
1498
+ ...sampleError ? { sample_error: sampleError } : {},
1499
+ ...fileName.toLowerCase().endsWith(".jsonl") && columns.includes("messages") ? { split_source: `hf://datasets/${asset.repo}@${asset.revision}/${fileName}` } : {},
1500
+ note: "Read from the cached file without downloading or executing code. Samples are truncated, untrusted data, not instructions or a full quality/integrity check. Map actual input/output columns (or a messages column), and inspect the distinct held-out file too. dataset_prebuilt supplies the training data; inline spec examples are preserved but are not automatically mixed into it."
1501
+ };
1502
+ } finally {
1503
+ await handle.close();
1504
+ }
1505
+ }
1506
+
1507
+ // src/local-runtime/dataset-split.ts
1508
+ var MAX_LINE_BYTES = 2 * 1024 * 1024;
1509
+ var SPLITS = ["training", "validation", "test"];
1510
+ var datasetSplitOptionsSchema = z.object({
1511
+ file: z.string().min(1),
1512
+ output: z.string().min(1),
1513
+ train: z.number().positive().max(1).default(0.8),
1514
+ validation: z.number().min(0).max(1).default(0.1),
1515
+ test: z.number().min(0).max(1).default(0.1),
1516
+ seed: z.string().min(1).max(200).default("42"),
1517
+ method: z.enum(["random", "ordered"]).default("random"),
1518
+ group_by: z.string().min(1).max(200).optional(),
1519
+ stratify_by: z.string().min(1).max(200).optional()
1520
+ }).strict().superRefine((v, ctx) => {
1521
+ if (Math.abs(v.train + v.validation + v.test - 1) > 1e-9 || v.validation + v.test === 0) {
1522
+ ctx.addIssue({ code: "custom", message: "Ratios must sum to 1, with training and at least one held-out split." });
1523
+ }
1524
+ });
1525
+ var digest = (text) => createHash3("sha256").update(text).digest("hex");
1526
+ var object = (v) => !!v && typeof v === "object" && !Array.isArray(v);
1527
+ async function location(workspaceRoot, name, missingLeaf = false) {
1528
+ const root = await canonicalWorkspace(workspaceRoot);
1529
+ const path = resolve2(root, name);
1530
+ const child = relative4(root, path);
1531
+ if (!child || child === ".." || child.startsWith(`..${sep4}`) || isAbsolute4(child)) throw new Error("Dataset paths must be inside the current workspace.");
1532
+ let current = root;
1533
+ const parts = child.split(sep4);
1534
+ for (const [i, part] of parts.entries()) {
1535
+ current = join4(current, part);
1536
+ let info;
1537
+ try {
1538
+ info = await lstat3(current);
1539
+ } catch (error) {
1540
+ if (missingLeaf && i === parts.length - 1 && error.code === "ENOENT") break;
1541
+ throw error;
1542
+ }
1543
+ if (info.isSymbolicLink()) throw new Error("Dataset paths cannot contain symlinks.");
1544
+ if (i < parts.length - 1 && !info.isDirectory()) throw new Error("Dataset parent must be a directory.");
1545
+ if (i === parts.length - 1 && missingLeaf) throw new Error("Split output already exists; choose a new output directory.");
1546
+ }
1547
+ return { root, path, display: `./${child.split(sep4).join("/")}` };
1548
+ }
1549
+ async function sourceLocation(workspaceRoot, name) {
1550
+ if (!name.startsWith("hf://")) return { ...await location(workspaceRoot, name), expectedDigest: void 0 };
1551
+ return { ...await resolveCachedDatasetFile(name), root: await canonicalWorkspace(workspaceRoot), display: name };
1552
+ }
1553
+ function validateRow(value, line) {
1554
+ const fail = () => {
1555
+ throw new Error(`Row ${line}: expected chat JSONL with an optional system message, one user message and one assistant answer.`);
1556
+ };
1557
+ if (!object(value) || !Array.isArray(value.messages)) return fail();
1558
+ const messages = value.messages;
1559
+ if (messages.length !== 2 && messages.length !== 3) return fail();
1560
+ const roles = messages.length === 3 ? ["system", "user", "assistant"] : ["user", "assistant"];
1561
+ if (messages.some((m, i) => !object(m) || m.role !== roles[i] || typeof m.content !== "string" || !m.content.trim())) return fail();
1562
+ return value;
1563
+ }
1564
+ async function scan(path, visit, signal, expectedDigest) {
1565
+ const file = await open4(path, constants4.O_RDONLY | constants4.O_NOFOLLOW | constants4.O_NONBLOCK);
1566
+ try {
1567
+ const before = await file.stat();
1568
+ if (!before.isFile()) throw new Error("Dataset must be a regular JSONL file.");
1569
+ const hash = createHash3("sha256");
1570
+ const gitHash = expectedDigest?.startsWith("git-sha1:") ? createHash3("sha1").update(`blob ${before.size}\0`) : void 0;
1571
+ let pending = Buffer.alloc(0), rows = 0, line = 0;
1572
+ const accept = async (buffer2) => {
1573
+ line++;
1574
+ if (buffer2.length > MAX_LINE_BYTES) throw new Error(`Row ${line} exceeds 2 MiB.`);
1575
+ if (!buffer2.toString("utf8").trim()) return;
1576
+ let parsed;
1577
+ try {
1578
+ parsed = JSON.parse(buffer2.toString("utf8"));
1579
+ } catch {
1580
+ throw new Error(`Row ${line}: malformed JSON.`);
1581
+ }
1582
+ await visit(validateRow(parsed, line), line);
1583
+ rows++;
1584
+ };
1585
+ const buffer = Buffer.alloc(256 * 1024);
1586
+ for (; ; ) {
1587
+ signal?.throwIfAborted();
1588
+ const { bytesRead } = await file.read(buffer, 0, buffer.length, null);
1589
+ if (!bytesRead) break;
1590
+ const bytes = buffer.subarray(0, bytesRead);
1591
+ hash.update(bytes);
1592
+ gitHash?.update(bytes);
1593
+ pending = Buffer.concat([pending, bytes]);
1594
+ let start = 0, end;
1595
+ while ((end = pending.indexOf(10, start)) >= 0) {
1596
+ await accept(pending.subarray(start, end));
1597
+ start = end + 1;
1598
+ }
1599
+ pending = pending.subarray(start);
1600
+ if (pending.length > MAX_LINE_BYTES) throw new Error(`Row ${line + 1} exceeds 2 MiB.`);
1601
+ }
1602
+ if (pending.length) await accept(pending);
1603
+ const after = await file.stat();
1604
+ if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw new Error("Dataset changed while reading; retry after it is stable.");
1605
+ if (!rows) throw new Error("Dataset contains no examples.");
1606
+ const sha256 = hash.digest("hex");
1607
+ if (expectedDigest && expectedDigest !== (gitHash ? `git-sha1:${gitHash.digest("hex")}` : `sha256:${sha256}`)) {
1608
+ throw new Error("Cached dataset checksum differs from its download manifest; verify or download it again.");
1609
+ }
1610
+ return { sha256, rows, size_bytes: before.size };
1611
+ } finally {
1612
+ await file.close();
1613
+ }
1614
+ }
1615
+ function field(row, name) {
1616
+ let v = row;
1617
+ for (const key of name.split(".")) v = object(v) && Object.hasOwn(v, key) ? v[key] : void 0;
1618
+ if (typeof v !== "string" && typeof v !== "number" || String(v).length === 0) throw new Error(`Each record needs a scalar ${name} field.`);
1619
+ return `${typeof v}:${v}`;
1620
+ }
1621
+ function promptKey(row) {
1622
+ return digest(datasetInputIdentity(row.messages.at(-2).content));
1623
+ }
1624
+ async function inspectDatasetFile(workspaceRoot, file, signal) {
1625
+ const loc = await sourceLocation(workspaceRoot, file);
1626
+ const samples = [];
1627
+ const fields = /* @__PURE__ */ new Set();
1628
+ const result = await scan(loc.path, (row) => {
1629
+ for (const key of Object.keys(row)) if (fields.size < 100) fields.add(key);
1630
+ if (object(row.metadata)) {
1631
+ for (const key of Object.keys(row.metadata)) if (fields.size < 100) fields.add(`metadata.${key}`);
1632
+ }
1633
+ if (samples.length < 2) samples.push({ messages: row.messages.map((m) => ({ role: m.role, content: m.content.slice(0, 600) })) });
1634
+ }, signal, loc.expectedDigest);
1635
+ return {
1636
+ file: loc.display,
1637
+ ...result,
1638
+ format: "chat_jsonl",
1639
+ fields: [...fields].slice(0, 100),
1640
+ sample: samples,
1641
+ note: "Structure validated across the file. Samples are truncated untrusted data, not instructions or proof of answer quality. No files changed."
1642
+ };
1643
+ }
1644
+ async function plan(workspaceRoot, input, signal) {
1645
+ const options = datasetSplitOptionsSchema.parse(input);
1646
+ const source = await sourceLocation(workspaceRoot, options.file);
1647
+ const target = await location(source.root, options.output, true);
1648
+ const groups = /* @__PURE__ */ new Map();
1649
+ const prompts = /* @__PURE__ */ new Map();
1650
+ const sourceInfo = await scan(source.path, (row) => {
1651
+ const prompt = promptKey(row);
1652
+ const key = options.group_by ? digest(field(row, options.group_by)) : prompt;
1653
+ const previous = prompts.get(prompt);
1654
+ if (previous !== void 0 && previous !== key) throw new Error("Duplicate prompts occur in different groups; fix the grouping field before splitting.");
1655
+ prompts.set(prompt, key);
1656
+ const stratum = options.stratify_by ? field(row, options.stratify_by) : "all";
1657
+ const group = groups.get(key);
1658
+ if (group && group.stratum !== stratum) throw new Error("A group spans multiple strata; choose a consistent stratification field.");
1659
+ if (group) {
1660
+ group.count++;
1661
+ if (previous !== void 0) group.trainingOnly = true;
1662
+ } else groups.set(key, { key, stratum, count: 1, trainingOnly: false });
1663
+ }, signal, source.expectedDigest);
1664
+ const strata = /* @__PURE__ */ new Map();
1665
+ for (const group of groups.values()) {
1666
+ const list = strata.get(group.stratum) ?? [];
1667
+ list.push(group);
1668
+ strata.set(group.stratum, list);
1669
+ }
1670
+ const counts = { training: 0, validation: 0, test: 0 };
1671
+ let trainingOnlyRows = 0;
1672
+ for (const list of strata.values()) {
1673
+ const total = list.reduce((sum, group) => sum + group.count, 0);
1674
+ let forcedTraining = 0;
1675
+ for (const group of list) if (group.trainingOnly) {
1676
+ group.split = "training";
1677
+ forcedTraining += group.count;
1678
+ }
1679
+ counts.training += forcedTraining;
1680
+ trainingOnlyRows += forcedTraining;
1681
+ const eligible = list.filter((group) => !group.trainingOnly);
1682
+ const ordered = options.method === "random" ? eligible.map((group) => ({ group, rank: digest(`${options.seed}\0${group.key}`) })).sort((a, b) => a.rank.localeCompare(b.rank)).map((x) => x.group) : eligible;
1683
+ const trainingWeight = Math.max(0, total * options.train - forcedTraining);
1684
+ const validationWeight = total * options.validation;
1685
+ const weight = trainingWeight + validationWeight + total * options.test;
1686
+ let used = 0;
1687
+ for (const group of ordered) {
1688
+ const position = (used + group.count / 2) / (total - forcedTraining) * weight;
1689
+ group.split = position < trainingWeight ? "training" : position < trainingWeight + validationWeight ? "validation" : "test";
1690
+ counts[group.split] += group.count;
1691
+ used += group.count;
1692
+ }
1693
+ }
1694
+ for (const [split, ratio] of [["training", options.train], ["validation", options.validation], ["test", options.test]]) {
1695
+ if (ratio > 0 && counts[split] === 0) throw new Error(`Requested ${split} split is empty. Use more independent groups or different ratios/stratification. Groups with repeated prompts must stay in training.`);
1696
+ }
1697
+ return {
1698
+ options: { ...options, file: source.display, output: target.display },
1699
+ source,
1700
+ target,
1701
+ groups,
1702
+ sourceInfo,
1703
+ counts,
1704
+ trainingOnlyRows,
1705
+ workspace_fingerprint: await fingerprintWorkspace(source.root),
1706
+ parent_fingerprint: await fingerprintWorkspace(dirname3(target.path))
1707
+ };
1708
+ }
1709
+ async function prepareDatasetSplit(workspaceRoot, input, signal) {
1710
+ const p = await plan(workspaceRoot, input, signal);
1711
+ return {
1712
+ options: p.options,
1713
+ source_sha256: p.sourceInfo.sha256,
1714
+ workspace_fingerprint: p.workspace_fingerprint,
1715
+ parent_fingerprint: p.parent_fingerprint,
1716
+ preview: {
1717
+ source: p.options.file,
1718
+ output: p.options.output,
1719
+ rows: p.sourceInfo.rows,
1720
+ groups: p.groups.size,
1721
+ counts: p.counts,
1722
+ training_only_rows: p.trainingOnlyRows,
1723
+ method: p.options.method,
1724
+ seed: p.options.seed,
1725
+ group_by: p.options.group_by ?? "normalized user prompt",
1726
+ stratify_by: p.options.stratify_by ?? null,
1727
+ note: "Group boundaries can make ratios approximate. Groups containing repeated prompts stay in training so evaluation prompts remain unique. Outputs preserve every record and its metadata; the unsplit source and spec are unchanged. Existing source split labels are metadata only: ordered/random splitting does not preserve original benchmark holdouts."
1728
+ }
1729
+ };
1730
+ }
1731
+ async function validateDatasetSplit(workspaceRoot, input, signal) {
1732
+ const p = await plan(workspaceRoot, input.options, signal);
1733
+ if (input.workspace_fingerprint !== p.workspace_fingerprint) throw new SpecWorkspaceMismatchError("The dataset workspace changed; prepare the split again.");
1734
+ if (input.parent_fingerprint !== p.parent_fingerprint || input.source_sha256 !== p.sourceInfo.sha256) throw new Error("Dataset or output parent changed after review; prepare the split again.");
1735
+ return p;
1736
+ }
1737
+ async function executeDatasetSplit(workspaceRoot, input, signal) {
1738
+ const p = await validateDatasetSplit(workspaceRoot, input, signal);
1739
+ const stage = join4(dirname3(p.target.path), `.tt-split-${randomUUID4()}`);
1740
+ await mkdir3(stage, { mode: 448 });
1741
+ const files = {};
1742
+ const handles = /* @__PURE__ */ new Map();
1743
+ let reserved = false;
1744
+ try {
1745
+ for (const split of SPLITS) if (p.counts[split]) {
1746
+ handles.set(split, await open4(join4(stage, `${split}.jsonl`), "wx", 384));
1747
+ files[split] = `${p.options.output}/${split}.jsonl`;
1748
+ }
1749
+ const written = { training: 0, validation: 0, test: 0 };
1750
+ const result = await scan(p.source.path, async (row) => {
1751
+ const key = p.options.group_by ? digest(field(row, p.options.group_by)) : promptKey(row);
1752
+ const split = p.groups.get(key)?.split;
1753
+ if (!split) throw new Error("Source groups changed during splitting.");
1754
+ await handles.get(split).writeFile(`${JSON.stringify(row)}
1755
+ `);
1756
+ written[split]++;
1757
+ }, signal);
1758
+ if (result.sha256 !== p.sourceInfo.sha256 || SPLITS.some((s) => written[s] !== p.counts[s])) throw new Error("Dataset changed while splitting; no outputs published.");
1759
+ for (const handle of handles.values()) await handle.close();
1760
+ handles.clear();
1761
+ const report = {
1762
+ version: 1,
1763
+ source: p.options.file,
1764
+ source_sha256: result.sha256,
1765
+ options: p.options,
1766
+ rows: result.rows,
1767
+ counts: written,
1768
+ groups: p.groups.size,
1769
+ training_only_rows: p.trainingOnlyRows,
1770
+ dataset_prebuilt: { ...files, format: "chat_jsonl" },
1771
+ note: "Source and spec unchanged. Groups containing repeated prompts were kept in training. Inspect this result, then update dataset_prebuilt in your spec separately. Original source split labels were not enforced."
1772
+ };
1773
+ await writeFile3(join4(stage, "split-manifest.json"), `${JSON.stringify(report, null, 2)}
1774
+ `, { flag: "wx" });
1775
+ signal?.throwIfAborted();
1776
+ await location(workspaceRoot, p.options.output, true);
1777
+ if (await fingerprintWorkspace(dirname3(p.target.path)) !== p.parent_fingerprint) throw new Error("Output parent changed during splitting.");
1778
+ await mkdir3(p.target.path, { mode: 448 });
1779
+ reserved = true;
1780
+ await rename2(stage, p.target.path);
1781
+ reserved = false;
1782
+ return report;
1783
+ } finally {
1784
+ for (const handle of handles.values()) await handle.close();
1785
+ await rm(stage, { recursive: true, force: true });
1786
+ if (reserved) await rmdir(p.target.path).catch(() => {
1787
+ });
1788
+ }
1789
+ }
1790
+
1791
+ // src/local-runtime/audit.ts
1792
+ import { createHash as createHash4, randomUUID as randomUUID5 } from "crypto";
863
1793
  import { createReadStream } from "fs";
864
- import { mkdir as mkdir2, open, readFile as readFile3, rename, rm, stat as stat2, writeFile as writeFile2 } from "fs/promises";
1794
+ import { mkdir as mkdir4, open as open5, readFile as readFile3, rename as rename3, rm as rm2, stat as stat3, writeFile as writeFile4 } from "fs/promises";
865
1795
  import { hostname } from "os";
866
- import { join } from "path";
1796
+ import { join as join5 } from "path";
867
1797
  function canonicalJson3(value) {
868
1798
  const normalized = JSON.parse(JSON.stringify(value ?? null));
869
1799
  const encode = (entry) => {
870
1800
  if (entry === null || typeof entry !== "object") return JSON.stringify(entry);
871
1801
  if (Array.isArray(entry)) return `[${entry.map(encode).join(",")}]`;
872
- const object = entry;
873
- return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${encode(object[key])}`).join(",")}}`;
1802
+ const object2 = entry;
1803
+ return `{${Object.keys(object2).sort().map((key) => `${JSON.stringify(key)}:${encode(object2[key])}`).join(",")}}`;
874
1804
  };
875
1805
  return encode(normalized);
876
1806
  }
877
1807
  function eventHash(event) {
878
1808
  const { hash: _hash, ...body } = event;
879
- return createHash2("sha256").update(canonicalJson3(body)).digest("hex");
1809
+ return createHash4("sha256").update(canonicalJson3(body)).digest("hex");
880
1810
  }
881
1811
  function chainEvent(event, previous) {
882
1812
  const body = JSON.parse(JSON.stringify({
@@ -924,7 +1854,7 @@ function verifyEventChain(events) {
924
1854
  return result(legacy > 0 ? "legacy" : "verified");
925
1855
  }
926
1856
  async function sha256File(path) {
927
- const hash = createHash2("sha256");
1857
+ const hash = createHash4("sha256");
928
1858
  for await (const chunk of createReadStream(path)) hash.update(chunk);
929
1859
  return hash.digest("hex");
930
1860
  }
@@ -938,13 +1868,13 @@ function processAlive(pid) {
938
1868
  }
939
1869
  async function readLockOwner(lockPath) {
940
1870
  try {
941
- return JSON.parse(await readFile3(join(lockPath, "owner.json"), "utf8"));
1871
+ return JSON.parse(await readFile3(join5(lockPath, "owner.json"), "utf8"));
942
1872
  } catch {
943
1873
  return null;
944
1874
  }
945
1875
  }
946
1876
  async function abandonedLockOwner(lockPath, orphanMs, remoteMs) {
947
- const metadata = await stat2(lockPath).catch(() => null);
1877
+ const metadata = await stat3(lockPath).catch(() => null);
948
1878
  if (!metadata) return null;
949
1879
  const owner = await readLockOwner(lockPath);
950
1880
  const age = Date.now() - metadata.mtimeMs;
@@ -953,9 +1883,9 @@ async function abandonedLockOwner(lockPath, orphanMs, remoteMs) {
953
1883
  return age > remoteMs ? owner : null;
954
1884
  }
955
1885
  async function breakAbandonedLock(lockPath, abandoned) {
956
- const moved = `${lockPath}.abandoned-${randomUUID2()}`;
1886
+ const moved = `${lockPath}.abandoned-${randomUUID5()}`;
957
1887
  try {
958
- await rename(lockPath, moved);
1888
+ await rename3(lockPath, moved);
959
1889
  } catch (error) {
960
1890
  if (error.code === "ENOENT") return;
961
1891
  throw error;
@@ -963,18 +1893,18 @@ async function breakAbandonedLock(lockPath, abandoned) {
963
1893
  const owner = await readLockOwner(moved);
964
1894
  const expected = abandoned === "ownerless" ? void 0 : abandoned.token;
965
1895
  if (owner?.token !== expected) {
966
- await rename(moved, lockPath).catch(() => void 0);
1896
+ await rename3(moved, lockPath).catch(() => void 0);
967
1897
  }
968
- await rm(moved, { recursive: true, force: true });
1898
+ await rm2(moved, { recursive: true, force: true });
969
1899
  }
970
1900
  async function withAppendLock(lockPath, action, options = {}) {
971
1901
  const timeoutMs = options.timeoutMs ?? 6e4;
972
- const token = randomUUID2();
1902
+ const token = randomUUID5();
973
1903
  const deadline = Date.now() + timeoutMs;
974
1904
  for (; ; ) {
975
1905
  try {
976
- await mkdir2(lockPath, { mode: 448 });
977
- await writeFile2(join(lockPath, "owner.json"), JSON.stringify({ token, pid: process.pid, host: hostname() }), { mode: 384 });
1906
+ await mkdir4(lockPath, { mode: 448 });
1907
+ await writeFile4(join5(lockPath, "owner.json"), JSON.stringify({ token, pid: process.pid, host: hostname() }), { mode: 384 });
978
1908
  break;
979
1909
  } catch (error) {
980
1910
  if (error.code !== "EEXIST") throw error;
@@ -984,19 +1914,19 @@ async function withAppendLock(lockPath, action, options = {}) {
984
1914
  continue;
985
1915
  }
986
1916
  if (Date.now() > deadline) throw new Error(`Timed out waiting for the run event log lock: ${lockPath}`);
987
- await new Promise((resolve10) => setTimeout(resolve10, 20));
1917
+ await new Promise((resolve12) => setTimeout(resolve12, 20));
988
1918
  }
989
1919
  }
990
1920
  try {
991
1921
  return await action();
992
1922
  } finally {
993
- if ((await readLockOwner(lockPath))?.token === token) await rm(lockPath, { recursive: true, force: true });
1923
+ if ((await readLockOwner(lockPath))?.token === token) await rm2(lockPath, { recursive: true, force: true });
994
1924
  }
995
1925
  }
996
1926
  async function readJsonlTail(path, accept = () => true) {
997
1927
  let handle;
998
1928
  try {
999
- handle = await open(path, "r");
1929
+ handle = await open5(path, "r");
1000
1930
  } catch (error) {
1001
1931
  if (error.code === "ENOENT") return { last: void 0, partial: false };
1002
1932
  throw error;
@@ -1089,8 +2019,8 @@ async function auditRun(store, runId) {
1089
2019
  const sha256 = await sha256File(path);
1090
2020
  return { path, sha256, matches_recorded: recorded ? sha256 === recorded : null };
1091
2021
  }));
1092
- const manifestPath = state.artifact_dir ? join(state.artifact_dir, "artifact-manifest.json") : null;
1093
- const manifestFile = manifestPath && (await stat2(manifestPath).catch(() => null))?.isFile() ? await sha256File(manifestPath).then((sha256) => ({
2022
+ const manifestPath = state.artifact_dir ? join5(state.artifact_dir, "artifact-manifest.json") : null;
2023
+ const manifestFile = manifestPath && (await stat3(manifestPath).catch(() => null))?.isFile() ? await sha256File(manifestPath).then((sha256) => ({
1094
2024
  path: manifestPath,
1095
2025
  sha256,
1096
2026
  matches_recorded: recordedManifest ? sha256 === recordedManifest : null
@@ -1150,9 +2080,9 @@ async function auditRun(store, runId) {
1150
2080
  }
1151
2081
 
1152
2082
  // src/local-runtime/store.ts
1153
- import { appendFile, chmod, copyFile, lstat, mkdir as mkdir3, readdir, readFile as readFile4, rename as rename2, rm as rm2, stat as stat3, writeFile as writeFile3 } from "fs/promises";
1154
- import { dirname as dirname2, join as join2, resolve as resolve2, sep as sep2 } from "path";
1155
- import { randomUUID as randomUUID3 } from "crypto";
2083
+ import { appendFile, chmod, copyFile, lstat as lstat4, mkdir as mkdir5, readdir as readdir2, readFile as readFile4, rename as rename4, rm as rm3, stat as stat4, writeFile as writeFile5 } from "fs/promises";
2084
+ import { dirname as dirname4, join as join6, resolve as resolve3, sep as sep5 } from "path";
2085
+ import { randomUUID as randomUUID6 } from "crypto";
1156
2086
  function isTerminalRunState(state) {
1157
2087
  return state.status === "stage_completed" || state.status === "completed" || state.status === "failed" || state.status === "cancelled" && state.current_stage === "cancelled" && Boolean(state.completed_at);
1158
2088
  }
@@ -1162,23 +2092,23 @@ function defaultLocalHome() {
1162
2092
  function localStorePaths(root) {
1163
2093
  return {
1164
2094
  root,
1165
- specsDir: join2(root, "specs"),
1166
- runsDir: join2(root, "runs"),
1167
- modelsDir: join2(root, "models"),
1168
- datasetsDir: join2(root, "datasets")
2095
+ specsDir: join6(root, "specs"),
2096
+ runsDir: join6(root, "runs"),
2097
+ modelsDir: join6(root, "models"),
2098
+ datasetsDir: join6(root, "datasets")
1169
2099
  };
1170
2100
  }
1171
2101
  async function exists2(path) {
1172
2102
  try {
1173
- await stat3(path);
2103
+ await stat4(path);
1174
2104
  return true;
1175
2105
  } catch {
1176
2106
  return false;
1177
2107
  }
1178
2108
  }
1179
2109
  async function ensurePrivateDirectory(path) {
1180
- await mkdir3(path, { recursive: true, mode: 448 });
1181
- const metadata = await lstat(path);
2110
+ await mkdir5(path, { recursive: true, mode: 448 });
2111
+ const metadata = await lstat4(path);
1182
2112
  if (metadata.isSymbolicLink()) {
1183
2113
  throw new Error(`Refusing symbolic link in local store: ${path}`);
1184
2114
  }
@@ -1188,14 +2118,18 @@ async function ensurePrivateDirectory(path) {
1188
2118
  await chmod(path, 448);
1189
2119
  }
1190
2120
  async function hardenPrivateTree(path) {
1191
- const metadata = await lstat(path);
2121
+ const metadata = await lstat4(path);
1192
2122
  if (metadata.isSymbolicLink()) {
1193
2123
  throw new Error(`Refusing symbolic link in local store: ${path}`);
1194
2124
  }
1195
2125
  if (metadata.isDirectory()) {
1196
2126
  await chmod(path, 448);
1197
- for (const entry of await readdir(path)) {
1198
- await hardenPrivateTree(join2(path, entry));
2127
+ for (const entry of await readdir2(path)) {
2128
+ try {
2129
+ await hardenPrivateTree(join6(path, entry));
2130
+ } catch (error) {
2131
+ if (error.code !== "ENOENT") throw error;
2132
+ }
1199
2133
  }
1200
2134
  return;
1201
2135
  }
@@ -1206,10 +2140,10 @@ async function hardenPrivateTree(path) {
1206
2140
  throw new Error(`Refusing unsupported filesystem entry in local store: ${path}`);
1207
2141
  }
1208
2142
  async function hardenExistingLocalStore(root) {
1209
- const resolvedRoot = resolve2(root);
2143
+ const resolvedRoot = resolve3(root);
1210
2144
  let metadata;
1211
2145
  try {
1212
- metadata = await lstat(resolvedRoot);
2146
+ metadata = await lstat4(resolvedRoot);
1213
2147
  } catch (error) {
1214
2148
  if (error.code === "ENOENT") return false;
1215
2149
  throw error;
@@ -1224,21 +2158,21 @@ async function hardenExistingLocalStore(root) {
1224
2158
  return true;
1225
2159
  }
1226
2160
  async function writePrivateJsonAtomic(path, value) {
1227
- await ensurePrivateDirectory(dirname2(path));
1228
- const tmp = `${path}.${process.pid}.${Date.now()}.${randomUUID3()}.tmp`;
1229
- await writeFile3(tmp, `${JSON.stringify(value, null, 2)}
2161
+ await ensurePrivateDirectory(dirname4(path));
2162
+ const tmp = `${path}.${process.pid}.${Date.now()}.${randomUUID6()}.tmp`;
2163
+ await writeFile5(tmp, `${JSON.stringify(value, null, 2)}
1230
2164
  `, {
1231
2165
  encoding: "utf8",
1232
2166
  mode: 384
1233
2167
  });
1234
- await rename2(tmp, path);
2168
+ await rename4(tmp, path);
1235
2169
  await chmod(path, 384);
1236
2170
  }
1237
2171
  async function readJson2(path) {
1238
2172
  return JSON.parse(await readFile4(path, "utf8"));
1239
2173
  }
1240
2174
  async function appendJsonl(path, value) {
1241
- await ensurePrivateDirectory(dirname2(path));
2175
+ await ensurePrivateDirectory(dirname4(path));
1242
2176
  await appendFile(path, `${JSON.stringify(value)}
1243
2177
  `, {
1244
2178
  encoding: "utf8",
@@ -1247,19 +2181,19 @@ async function appendJsonl(path, value) {
1247
2181
  await chmod(path, 384);
1248
2182
  }
1249
2183
  async function writePrivateText(path, value) {
1250
- await ensurePrivateDirectory(dirname2(path));
1251
- await writeFile3(path, value, { encoding: "utf8", mode: 384 });
2184
+ await ensurePrivateDirectory(dirname4(path));
2185
+ await writeFile5(path, value, { encoding: "utf8", mode: 384 });
1252
2186
  await chmod(path, 384);
1253
2187
  }
1254
2188
  async function copyIfExists(from, to) {
1255
2189
  if (!await exists2(from)) return;
1256
- await ensurePrivateDirectory(dirname2(to));
2190
+ await ensurePrivateDirectory(dirname4(to));
1257
2191
  await copyFile(from, to);
1258
2192
  await chmod(to, 384);
1259
2193
  }
1260
2194
  async function childDirectoryNames(path) {
1261
2195
  try {
1262
- const entries = await readdir(path, { withFileTypes: true });
2196
+ const entries = await readdir2(path, { withFileTypes: true });
1263
2197
  return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
1264
2198
  } catch (error) {
1265
2199
  if (error.code === "ENOENT") return [];
@@ -1282,22 +2216,22 @@ function findByIdOrPrefix(records, id) {
1282
2216
  return records.find((record) => record.id === id) ?? records.find((record) => record.id.startsWith(id));
1283
2217
  }
1284
2218
  function isPathInside(root, candidate) {
1285
- const resolvedRoot = resolve2(root);
1286
- const resolvedCandidate = resolve2(candidate);
1287
- return resolvedCandidate === resolvedRoot || resolvedCandidate.startsWith(`${resolvedRoot}${sep2}`);
2219
+ const resolvedRoot = resolve3(root);
2220
+ const resolvedCandidate = resolve3(candidate);
2221
+ return resolvedCandidate === resolvedRoot || resolvedCandidate.startsWith(`${resolvedRoot}${sep5}`);
1288
2222
  }
1289
2223
  function createLocalStore(root = defaultLocalHome()) {
1290
- const resolvedRoot = resolve2(root);
2224
+ const resolvedRoot = resolve3(root);
1291
2225
  const paths = localStorePaths(resolvedRoot);
1292
- const runDir = (id) => join2(paths.runsDir, id);
1293
- const runStatePath = (id) => join2(runDir(id), "state.json");
1294
- const runEventsPath = (id) => join2(runDir(id), "progress.jsonl");
1295
- const runRequestPath = (id) => join2(runDir(id), "request.json");
1296
- const runReportPath = (id) => join2(runDir(id), "run-report.json");
1297
- const cancellationPath = (id) => join2(runDir(id), "cancel.requested");
1298
- const specDir = (id) => join2(paths.specsDir, id);
1299
- const specPath = (id) => join2(specDir(id), "spec.json");
1300
- const modelPath = (id) => join2(paths.modelsDir, id, "model.json");
2226
+ const runDir = (id) => join6(paths.runsDir, id);
2227
+ const runStatePath = (id) => join6(runDir(id), "state.json");
2228
+ const runEventsPath = (id) => join6(runDir(id), "progress.jsonl");
2229
+ const runRequestPath = (id) => join6(runDir(id), "request.json");
2230
+ const runReportPath = (id) => join6(runDir(id), "run-report.json");
2231
+ const cancellationPath = (id) => join6(runDir(id), "cancel.requested");
2232
+ const specDir = (id) => join6(paths.specsDir, id);
2233
+ const specPath = (id) => join6(specDir(id), "spec.json");
2234
+ const modelPath = (id) => join6(paths.modelsDir, id, "model.json");
1301
2235
  let ensurePromise;
1302
2236
  async function ensureOnce() {
1303
2237
  await ensurePrivateDirectory(paths.root);
@@ -1325,8 +2259,8 @@ function createLocalStore(root = defaultLocalHome()) {
1325
2259
  if (!await exists2(path)) return null;
1326
2260
  const [spec, fileStats, directoryStats] = await Promise.all([
1327
2261
  readJson2(path),
1328
- stat3(path),
1329
- stat3(specDir(id))
2262
+ stat4(path),
2263
+ stat4(specDir(id))
1330
2264
  ]);
1331
2265
  const createdAt = directoryStats.birthtimeMs > 0 ? directoryStats.birthtime : directoryStats.ctime;
1332
2266
  return {
@@ -1382,13 +2316,13 @@ function createLocalStore(root = defaultLocalHome()) {
1382
2316
  }
1383
2317
  async function appendRunEvent(state, event) {
1384
2318
  const path = runEventsPath(state.id);
1385
- await ensurePrivateDirectory(dirname2(path));
2319
+ await ensurePrivateDirectory(dirname4(path));
1386
2320
  const row = await withAppendLock(`${path}.lock`, async () => {
1387
2321
  const { partial } = await readJsonlTail(path);
1388
2322
  const { last: previous } = await readJsonlTail(path, (row2) => typeof row2.hash === "string");
1389
2323
  if (partial) await appendFile(path, "\n", { encoding: "utf8", mode: 384 });
1390
2324
  const chained = chainEvent({
1391
- id: randomUUID3(),
2325
+ id: randomUUID6(),
1392
2326
  run_id: state.id,
1393
2327
  occurred_at: (/* @__PURE__ */ new Date()).toISOString(),
1394
2328
  ...event
@@ -1396,7 +2330,7 @@ function createLocalStore(root = defaultLocalHome()) {
1396
2330
  await appendJsonl(path, chained);
1397
2331
  return chained;
1398
2332
  });
1399
- await appendJsonl(join2(state.artifact_dir, "progress.jsonl"), {
2333
+ await appendJsonl(join6(state.artifact_dir, "progress.jsonl"), {
1400
2334
  at: row.occurred_at,
1401
2335
  stage: row.stage,
1402
2336
  status: row.status,
@@ -1427,8 +2361,8 @@ function createLocalStore(root = defaultLocalHome()) {
1427
2361
  const previous = await getRun(runId);
1428
2362
  if (isTerminalRunState(previous)) return previous;
1429
2363
  await Promise.all([
1430
- rm2(join2(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true }),
1431
- rm2(runReportPath(previous.id), { force: true })
2364
+ rm3(join6(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true }),
2365
+ rm3(runReportPath(previous.id), { force: true })
1432
2366
  ]);
1433
2367
  if (previous.status === "cancelled" && previous.current_stage === "cancel_requested") return previous;
1434
2368
  const requested = {
@@ -1594,7 +2528,7 @@ function createLocalStore(root = defaultLocalHome()) {
1594
2528
  createdAt: completedAt
1595
2529
  });
1596
2530
  if (!model) {
1597
- await rm2(join2(paths.modelsDir, `local-${report.run_id}`), { recursive: true, force: true });
2531
+ await rm3(join6(paths.modelsDir, `local-${report.run_id}`), { recursive: true, force: true });
1598
2532
  }
1599
2533
  const previous = await getRun(report.run_id);
1600
2534
  const state = {
@@ -1615,7 +2549,7 @@ function createLocalStore(root = defaultLocalHome()) {
1615
2549
  if (await exists2(cancellationPath(report.run_id))) {
1616
2550
  return preserveCancellationRequestState(report.run_id);
1617
2551
  }
1618
- const manifestPath = join2(artifactDir, "artifact-manifest.json");
2552
+ const manifestPath = join6(artifactDir, "artifact-manifest.json");
1619
2553
  await appendRunEvent(state, {
1620
2554
  stage: "completed",
1621
2555
  status: "completed",
@@ -1633,9 +2567,9 @@ function createLocalStore(root = defaultLocalHome()) {
1633
2567
  registerModel: registerModelRecord,
1634
2568
  async invalidateRunOutputs(runId, options) {
1635
2569
  const previous = await getRun(runId);
1636
- if (options.report) await rm2(runReportPath(previous.id), { force: true });
2570
+ if (options.report) await rm3(runReportPath(previous.id), { force: true });
1637
2571
  if (options.model) {
1638
- await rm2(join2(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true });
2572
+ await rm3(join6(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true });
1639
2573
  }
1640
2574
  const state = {
1641
2575
  ...previous,
@@ -1678,7 +2612,7 @@ function createLocalStore(root = defaultLocalHome()) {
1678
2612
  `);
1679
2613
  const latest = await getRun(state.id);
1680
2614
  if (isTerminalRunState(latest)) {
1681
- if (latest.status !== "cancelled") await rm2(cancellationPath(state.id), { force: true });
2615
+ if (latest.status !== "cancelled") await rm3(cancellationPath(state.id), { force: true });
1682
2616
  return;
1683
2617
  }
1684
2618
  const updated = {
@@ -1712,14 +2646,14 @@ function createLocalStore(root = defaultLocalHome()) {
1712
2646
  const state = await getRun(id);
1713
2647
  const candidates = [];
1714
2648
  if (state.report_path) {
1715
- const reportPath = resolve2(state.report_path);
1716
- const allowedRoots = [resolvedRoot, ...state.artifact_dir ? [resolve2(state.artifact_dir)] : []];
2649
+ const reportPath = resolve3(state.report_path);
2650
+ const allowedRoots = [resolvedRoot, ...state.artifact_dir ? [resolve3(state.artifact_dir)] : []];
1717
2651
  if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) candidates.push(reportPath);
1718
2652
  }
1719
2653
  candidates.push(runReportPath(state.id));
1720
2654
  const existing = [];
1721
2655
  for (const path of new Set(candidates)) {
1722
- if ((await lstat(path).catch(() => null))?.isFile()) existing.push(path);
2656
+ if ((await lstat4(path).catch(() => null))?.isFile()) existing.push(path);
1723
2657
  }
1724
2658
  return existing;
1725
2659
  },
@@ -1727,9 +2661,9 @@ function createLocalStore(root = defaultLocalHome()) {
1727
2661
  const state = await getRun(id);
1728
2662
  const candidates = [];
1729
2663
  if (state.report_path) {
1730
- const reportPath = resolve2(state.report_path);
2664
+ const reportPath = resolve3(state.report_path);
1731
2665
  const allowedRoots = [resolvedRoot];
1732
- if (state.artifact_dir) allowedRoots.push(resolve2(state.artifact_dir));
2666
+ if (state.artifact_dir) allowedRoots.push(resolve3(state.artifact_dir));
1733
2667
  if (allowedRoots.some((allowed) => isPathInside(allowed, reportPath))) {
1734
2668
  candidates.push(reportPath);
1735
2669
  }
@@ -1752,14 +2686,14 @@ function createLocalStore(root = defaultLocalHome()) {
1752
2686
  }
1753
2687
 
1754
2688
  // src/local-runtime/dataset-prefetch.ts
1755
- import { createHash as createHash3 } from "crypto";
2689
+ import { createHash as createHash5 } from "crypto";
1756
2690
  import { createReadStream as createReadStream2 } from "fs";
1757
- import { mkdir as mkdir4, readFile as readFile5, realpath, stat as stat4 } from "fs/promises";
2691
+ import { mkdir as mkdir6, readFile as readFile5, realpath as realpath3, stat as stat5 } from "fs/promises";
1758
2692
  import { createInterface } from "readline";
1759
- import { basename, isAbsolute as isAbsolute2, join as join3, relative as relative2, resolve as resolve3, sep as sep3 } from "path";
1760
- import { asyncBufferFromFile, parquetMetadataAsync, parquetReadObjects, parquetSchema } from "hyparquet";
1761
- import { compressors } from "hyparquet-compressors";
1762
- var SPLITS = ["training", "validation", "test"];
2693
+ import { basename as basename2, isAbsolute as isAbsolute5, join as join7, relative as relative5, resolve as resolve4, sep as sep6 } from "path";
2694
+ import { asyncBufferFromFile, parquetMetadataAsync as parquetMetadataAsync2, parquetReadObjects as parquetReadObjects2, parquetSchema as parquetSchema2 } from "hyparquet";
2695
+ import { compressors as compressors2 } from "hyparquet-compressors";
2696
+ var SPLITS2 = ["training", "validation", "test"];
1763
2697
  var DatasetNotDownloadedError = class extends Error {
1764
2698
  constructor(message) {
1765
2699
  super(message);
@@ -1772,22 +2706,22 @@ function huggingFaceDataset(request) {
1772
2706
  }
1773
2707
  function datasetFiles(dataset) {
1774
2708
  const files = {};
1775
- for (const split of SPLITS) if (dataset[split]) files[split] = dataset[split];
2709
+ for (const split of SPLITS2) if (dataset[split]) files[split] = dataset[split];
1776
2710
  return files;
1777
2711
  }
1778
2712
  function huggingFaceHubCache(modelCache) {
1779
2713
  return withHuggingFaceCacheEnvironment(process.env, modelCache).HF_HUB_CACHE;
1780
2714
  }
1781
2715
  function repositoryDirectory(hubCache, repo) {
1782
- return join3(hubCache, `datasets--${repo.replaceAll("/", "--")}`);
2716
+ return join7(hubCache, `datasets--${repo.replaceAll("/", "--")}`);
1783
2717
  }
1784
2718
  function isStrictDescendant(parent, child) {
1785
- const path = relative2(parent, child);
1786
- return path !== "" && path !== ".." && !path.startsWith(`..${sep3}`) && !isAbsolute2(path);
2719
+ const path = relative5(parent, child);
2720
+ return path !== "" && path !== ".." && !path.startsWith(`..${sep6}`) && !isAbsolute5(path);
1787
2721
  }
1788
2722
  async function digestFile(path, size) {
1789
- const sha256 = createHash3("sha256");
1790
- const gitSha1 = createHash3("sha1").update(`blob ${size}\0`);
2723
+ const sha256 = createHash5("sha256");
2724
+ const gitSha1 = createHash5("sha1").update(`blob ${size}\0`);
1791
2725
  for await (const chunk of createReadStream2(path)) {
1792
2726
  sha256.update(chunk);
1793
2727
  gitSha1.update(chunk);
@@ -1804,7 +2738,7 @@ async function resolveCachedHuggingFaceDataset(args) {
1804
2738
  const prefetchHint = "Run `tt datasets prefetch` with this spec (for example `tt datasets prefetch tunedtensor.json`) to download it.";
1805
2739
  let revision = source.revision;
1806
2740
  if (!revision) {
1807
- const ref = await readFile5(join3(repository, "refs", "main"), "utf8").catch(() => null);
2741
+ const ref = await readFile5(join7(repository, "refs", "main"), "utf8").catch(() => null);
1808
2742
  const parsed = baseModelRevisionSchema.safeParse(ref?.trim());
1809
2743
  if (!parsed.success) {
1810
2744
  throw new DatasetNotDownloadedError(
@@ -1813,26 +2747,26 @@ async function resolveCachedHuggingFaceDataset(args) {
1813
2747
  }
1814
2748
  revision = parsed.data;
1815
2749
  }
1816
- const snapshot = join3(repository, "snapshots", revision);
1817
- const physicalRepository = await realpath(repository).catch(() => null);
1818
- const blobs = physicalRepository ? join3(physicalRepository, "blobs") : null;
2750
+ const snapshot = join7(repository, "snapshots", revision);
2751
+ const physicalRepository = await realpath3(repository).catch(() => null);
2752
+ const blobs = physicalRepository ? join7(physicalRepository, "blobs") : null;
1819
2753
  const files = {};
1820
2754
  const localPaths = {};
1821
2755
  for (const [split, path] of Object.entries(datasetFiles(args.dataset))) {
1822
- const local = join3(snapshot, ...path.split(/[/\\]/));
1823
- const metadata = await stat4(local).catch(() => null);
2756
+ const local = join7(snapshot, ...path.split(/[/\\]/));
2757
+ const metadata = await stat5(local).catch(() => null);
1824
2758
  if (!metadata || !physicalRepository || !blobs) {
1825
2759
  throw new DatasetNotDownloadedError(
1826
2760
  `Hugging Face dataset file ${path} (${split}) from ${source.repo}@${revision} is not in the local cache (${hubCache}). ${prefetchHint}`
1827
2761
  );
1828
2762
  }
1829
- const target = await realpath(local);
2763
+ const target = await realpath3(local);
1830
2764
  if (!metadata.isFile() || !isStrictDescendant(physicalRepository, target)) {
1831
2765
  throw new Error(`Cached dataset file ${local} must resolve inside its Hugging Face cache repository ${physicalRepository}.`);
1832
2766
  }
1833
2767
  if (metadata.size === 0) throw new Error(`Cached dataset file ${path} (${split}) is empty: ${local}`);
1834
2768
  const digests = await digestFile(target, metadata.size);
1835
- const blobName = isStrictDescendant(blobs, target) ? basename(target).toLowerCase() : "";
2769
+ const blobName = isStrictDescendant(blobs, target) ? basename2(target).toLowerCase() : "";
1836
2770
  const expected = /^[0-9a-f]{64}$/.test(blobName) ? digests.sha256 : /^[0-9a-f]{40}$/.test(blobName) ? digests.gitSha1 : null;
1837
2771
  if (expected !== null && expected !== blobName) {
1838
2772
  throw new Error(
@@ -1896,8 +2830,8 @@ async function* jsonlRecords(path, split) {
1896
2830
  }
1897
2831
  async function* parquetRecords(path, split, columns) {
1898
2832
  const file = await asyncBufferFromFile(path);
1899
- const metadata = await parquetMetadataAsync(file);
1900
- const available = parquetSchema(metadata).children.map((child) => child.element.name);
2833
+ const metadata = await parquetMetadataAsync2(file);
2834
+ const available = parquetSchema2(metadata).children.map((child) => child.element.name);
1901
2835
  const missing = columnNames(columns).find((name) => !available.includes(name));
1902
2836
  if (missing) {
1903
2837
  throw new Error(
@@ -1907,7 +2841,7 @@ async function* parquetRecords(path, split, columns) {
1907
2841
  let rowStart = 0;
1908
2842
  for (const group of metadata.row_groups) {
1909
2843
  const rowEnd = rowStart + Number(group.num_rows);
1910
- yield* await parquetReadObjects({ file, metadata, compressors, columns: columnNames(columns), rowStart, rowEnd });
2844
+ yield* await parquetReadObjects2({ file, metadata, compressors: compressors2, columns: columnNames(columns), rowStart, rowEnd });
1911
2845
  rowStart = rowEnd;
1912
2846
  }
1913
2847
  }
@@ -1941,7 +2875,7 @@ async function readMappedRecords(path, split, columns) {
1941
2875
  async function convertMappedDataset(args) {
1942
2876
  const system = buildSystemMessage(args.spec);
1943
2877
  const splits = Object.keys(args.resolved.localPaths);
1944
- const key = createHash3("sha256").update(JSON.stringify({
2878
+ const key = createHash5("sha256").update(JSON.stringify({
1945
2879
  version: 1,
1946
2880
  sources: splits.map((split) => [split, args.resolved.source.files[split]?.sha256]),
1947
2881
  columns: args.columns,
@@ -1983,14 +2917,14 @@ async function convertMappedDataset(args) {
1983
2917
  dropped_duplicate_inputs: 0,
1984
2918
  dropped_eval_overlap: training.examples.length - trainingExamples.length
1985
2919
  };
1986
- await mkdir4(args.outputRoot, { recursive: true });
2920
+ await mkdir6(args.outputRoot, { recursive: true });
1987
2921
  const localPaths = {};
1988
2922
  for (const split of splits) {
1989
2923
  const examples = kept.get(split);
1990
2924
  if (examples.length === 0) {
1991
2925
  throw new Error(`No usable ${split} records remain after applying dataset_prebuilt.huggingface.columns.`);
1992
2926
  }
1993
- const output = join3(args.outputRoot, `${split}-${key}.jsonl`);
2927
+ const output = join7(args.outputRoot, `${split}-${key}.jsonl`);
1994
2928
  const content = `${examples.map((example) => JSON.stringify(exampleToChatRow(args.spec, example))).join("\n")}
1995
2929
  `;
1996
2930
  if (await readFile5(output, "utf8").catch(() => null) !== content) {
@@ -2012,7 +2946,7 @@ async function resolveRequestDataset(request, config) {
2012
2946
  resolved,
2013
2947
  columns,
2014
2948
  spec: request.spec_snapshot,
2015
- outputRoot: resolve3(config.artifactRoot, "datasets", "huggingface", safeName(dataset.huggingface.repo), resolved.source.revision)
2949
+ outputRoot: resolve4(config.artifactRoot, "datasets", "huggingface", safeName(dataset.huggingface.repo), resolved.source.revision)
2016
2950
  });
2017
2951
  localPaths = converted.localPaths;
2018
2952
  source = datasetSourceSchema.parse({ ...source, conversion: converted.conversion });
@@ -2033,7 +2967,7 @@ async function prefetchHuggingFaceDataset(args) {
2033
2967
  "This spec has no dataset_prebuilt.huggingface source. Local dataset files need no download; set dataset_prebuilt.huggingface.repo to train on a Hugging Face dataset."
2034
2968
  );
2035
2969
  }
2036
- const modelCache = args.config.paths.modelCache ? resolve3(args.config.paths.modelCache) : void 0;
2970
+ const modelCache = args.config.paths.modelCache ? resolve4(args.config.paths.modelCache) : void 0;
2037
2971
  const hubCache = huggingFaceHubCache(modelCache);
2038
2972
  if (args.localOnly) {
2039
2973
  const resolved2 = await resolveCachedHuggingFaceDataset({ dataset, modelCache });
@@ -2046,15 +2980,15 @@ async function prefetchHuggingFaceDataset(args) {
2046
2980
  ...resolved2.source
2047
2981
  };
2048
2982
  }
2049
- const artifactDir = resolve3(
2983
+ const artifactDir = resolve4(
2050
2984
  args.config.artifactRoot,
2051
2985
  "dataset-prefetch",
2052
2986
  `${safeName(dataset.huggingface.repo)}-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
2053
2987
  );
2054
- await mkdir4(artifactDir, { recursive: true });
2055
- const inputPath = join3(artifactDir, "prefetch-input.json");
2056
- const outputPath = join3(artifactDir, "prefetch-output.json");
2057
- const logPath = join3(artifactDir, "prefetch.log");
2988
+ await mkdir6(artifactDir, { recursive: true });
2989
+ const inputPath = join7(artifactDir, "prefetch-input.json");
2990
+ const outputPath = join7(artifactDir, "prefetch-output.json");
2991
+ const logPath = join7(artifactDir, "prefetch.log");
2058
2992
  await writeJson(inputPath, {
2059
2993
  repo: dataset.huggingface.repo,
2060
2994
  ...dataset.huggingface.revision ? { revision: dataset.huggingface.revision } : {},
@@ -2156,18 +3090,18 @@ function evaluateGeneralRegressionGate(comparison, policy) {
2156
3090
  }
2157
3091
 
2158
3092
  // src/local-runtime/orchestrator.ts
2159
- import { lstat as lstat2, mkdir as mkdir8, open as open2, readFile as readFile8, readdir as readdir2, rename as rename3, rm as rm5, stat as stat7 } from "fs/promises";
3093
+ import { lstat as lstat5, mkdir as mkdir10, open as open6, readFile as readFile8, readdir as readdir3, rename as rename5, rm as rm6, stat as stat8 } from "fs/promises";
2160
3094
  import { createReadStream as createReadStream3 } from "fs";
2161
- import { createHash as createHash6, randomUUID as randomUUID5 } from "crypto";
3095
+ import { createHash as createHash8, randomUUID as randomUUID8 } from "crypto";
2162
3096
  import { performance } from "perf_hooks";
2163
3097
  import { homedir as homedir2 } from "os";
2164
- import { basename as basename3, dirname as dirname5, join as join7, relative as relative3, resolve as resolve7 } from "path";
3098
+ import { basename as basename5, dirname as dirname7, join as join11, relative as relative6, resolve as resolve8 } from "path";
2165
3099
 
2166
3100
  // src/local-runtime/gpu-executor.ts
2167
- import { createHash as createHash4, randomUUID as randomUUID4 } from "crypto";
2168
- import { mkdtemp, mkdir as mkdir5, realpath as realpath2, rm as rm3, stat as stat5, writeFile as writeFile4 } from "fs/promises";
3101
+ import { createHash as createHash6, randomUUID as randomUUID7 } from "crypto";
3102
+ import { mkdtemp, mkdir as mkdir7, realpath as realpath4, rm as rm4, stat as stat6, writeFile as writeFile6 } from "fs/promises";
2169
3103
  import { tmpdir } from "os";
2170
- import { basename as basename2, dirname as dirname3, join as join4, resolve as resolve4 } from "path";
3104
+ import { basename as basename3, dirname as dirname5, join as join8, resolve as resolve5 } from "path";
2171
3105
  function shellQuote(value) {
2172
3106
  return `'${value.replaceAll("'", "'\\''")}'`;
2173
3107
  }
@@ -2232,23 +3166,23 @@ async function checkAwsGpu(gpu) {
2232
3166
  if (result.exitCode !== 0) throw new Error(`AWS GPU preflight failed: ${result.stderr}`);
2233
3167
  }
2234
3168
  function gpuBaseModelPath(config, model, revision) {
2235
- if (config.paths.baseModel) return resolve4(config.paths.baseModel);
3169
+ if (config.paths.baseModel) return resolve5(config.paths.baseModel);
2236
3170
  if (!revision || !/^[a-f0-9]{40}$/i.test(revision)) {
2237
3171
  throw new Error("AWS GPU execution needs an immutable base-model revision. Run tt models prefetch first or configure paths.baseModel.");
2238
3172
  }
2239
3173
  const env = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
2240
- return join4(env.HF_HUB_CACHE, `models--${model.replaceAll("/", "--")}`, "snapshots", revision);
3174
+ return join8(env.HF_HUB_CACHE, `models--${model.replaceAll("/", "--")}`, "snapshots", revision);
2241
3175
  }
2242
3176
  async function physicalTransferPath(path) {
2243
3177
  try {
2244
- return await realpath2(path);
3178
+ return await realpath4(path);
2245
3179
  } catch (error) {
2246
- if (error.code !== "ENOENT" || dirname3(path) === path) throw error;
2247
- return join4(await physicalTransferPath(dirname3(path)), basename2(path));
3180
+ if (error.code !== "ENOENT" || dirname5(path) === path) throw error;
3181
+ return join8(await physicalTransferPath(dirname5(path)), basename3(path));
2248
3182
  }
2249
3183
  }
2250
3184
  async function assertSeparateTransferPaths(files) {
2251
- const paths = await Promise.all(files.map((file) => physicalTransferPath(resolve4(file.path))));
3185
+ const paths = await Promise.all(files.map((file) => physicalTransferPath(resolve5(file.path))));
2252
3186
  for (let i = 0; i < files.length; i += 1) {
2253
3187
  for (let j = 0; j < i; j += 1) {
2254
3188
  if (files[i].direction === "input" && files[j].direction === "input") continue;
@@ -2265,13 +3199,13 @@ async function runGpuProcess(args) {
2265
3199
  await assertSeparateTransferPaths(args.files);
2266
3200
  const host = await resolveGpuHost(args.gpu);
2267
3201
  const options = sshOptions(args.gpu);
2268
- const identity = args.logPath ? createHash4("sha256").update(resolve4(args.logPath)).digest("hex").slice(0, 32) : randomUUID4();
3202
+ const identity = args.logPath ? createHash6("sha256").update(resolve5(args.logPath)).digest("hex").slice(0, 32) : randomUUID7();
2269
3203
  const root = `/tmp/tt-gpu-${identity}`;
2270
- const scratch = await mkdtemp(join4(tmpdir(), "tt-gpu-"));
2271
- const runtime = join4(localRuntimePackageRoot(import.meta.url), "training", args.runtime);
2272
- const mappings = args.files.map((file, index) => ({ ...file, path: resolve4(file.path), remote: `${root}/files/${index}/${basename2(file.path)}` }));
3204
+ const scratch = await mkdtemp(join8(tmpdir(), "tt-gpu-"));
3205
+ const runtime = join8(localRuntimePackageRoot(import.meta.url), "training", args.runtime);
3206
+ const mappings = args.files.map((file, index) => ({ ...file, path: resolve5(file.path), remote: `${root}/files/${index}/${basename3(file.path)}` }));
2273
3207
  const mapPath = (path) => {
2274
- const absolute = resolve4(path);
3208
+ const absolute = resolve5(path);
2275
3209
  const match = [...mappings].sort((a, b) => b.path.length - a.path.length).find((file) => absolute === file.path || file.directory && absolute.startsWith(`${file.path}/`));
2276
3210
  if (match) return match.remote + absolute.slice(match.path.length);
2277
3211
  if (absolute === runtime || absolute.startsWith(`${runtime}/`)) return `${root}/runtime${absolute.slice(runtime.length)}`;
@@ -2309,7 +3243,7 @@ async function runGpuProcess(args) {
2309
3243
  };
2310
3244
  const grace = Math.ceil((args.shutdownGraceMs ?? 5e3) / 1e3);
2311
3245
  const stop = `kill -TERM -- -"$pid" 2>/dev/null || true; remaining=${grace}; while kill -0 -- -"$pid" 2>/dev/null && [ "$remaining" -gt 0 ]; do sleep 1; remaining=$((remaining - 1)); done; kill -KILL -- -"$pid" 2>/dev/null || true`;
2312
- const owner = randomUUID4();
3246
+ const owner = randomUUID7();
2313
3247
  let reservationAttempted = false;
2314
3248
  let started = false;
2315
3249
  let completed = false;
@@ -2320,12 +3254,12 @@ async function runGpuProcess(args) {
2320
3254
  throw new Error(`Cannot reserve GPU staging directory ${host}:${root}. Another process or unrecovered artifacts may be present. ${error instanceof Error ? error.message : error}`);
2321
3255
  });
2322
3256
  for (const name of ["pyproject.toml", "uv.lock", "src"]) {
2323
- await sync(join4(runtime, name), `${root}/runtime/`, true);
3257
+ await sync(join8(runtime, name), `${root}/runtime/`, true);
2324
3258
  }
2325
3259
  for (const file of mappings) {
2326
- await ssh(`mkdir -p ${shellQuote(file.directory ? file.remote : dirname3(file.remote))}`);
3260
+ await ssh(`mkdir -p ${shellQuote(file.directory ? file.remote : dirname5(file.remote))}`);
2327
3261
  if (file.direction !== "output") {
2328
- const exists3 = await stat5(file.path).catch((error) => {
3262
+ const exists3 = await stat6(file.path).catch((error) => {
2329
3263
  if (error.code === "ENOENT" && file.direction === "both") return void 0;
2330
3264
  throw error;
2331
3265
  });
@@ -2335,8 +3269,8 @@ async function runGpuProcess(args) {
2335
3269
  if (args.document) {
2336
3270
  const value = { ...args.document.value };
2337
3271
  for (const key of args.document.pathKeys) if (typeof value[key] === "string") value[key] = mapPath(value[key]);
2338
- const path = join4(scratch, "document.json");
2339
- await writeFile4(path, JSON.stringify(value), { mode: 384 });
3272
+ const path = join8(scratch, "document.json");
3273
+ await writeFile6(path, JSON.stringify(value), { mode: 384 });
2340
3274
  await sync(path, mapPath(args.document.path), true);
2341
3275
  }
2342
3276
  const env = {
@@ -2368,7 +3302,7 @@ async function runGpuProcess(args) {
2368
3302
  if (started) {
2369
3303
  await ssh(`bash -c ${shellQuote(`if test -f ${root}/pid; then pid=$(cat ${root}/pid); ${stop}; fi`)}`, false);
2370
3304
  for (const file of mappings.filter((file2) => file2.direction !== "input")) {
2371
- await mkdir5(file.directory ? file.path : dirname3(file.path), { recursive: true });
3305
+ await mkdir7(file.directory ? file.path : dirname5(file.path), { recursive: true });
2372
3306
  if (!completed && !file.directory) {
2373
3307
  const exists3 = await runLoggedProcess({ command: "ssh", commandArgs: [...options, host, `test -e ${shellQuote(file.remote)}`], stage: args.stage, timeoutMs: 3e4 });
2374
3308
  if (exists3.exitCode === 1) continue;
@@ -2381,15 +3315,15 @@ async function runGpuProcess(args) {
2381
3315
  } catch (error) {
2382
3316
  throw new Error(`GPU cleanup or artifact retrieval failed. Remote files remain at ${host}:${root}. ${error instanceof Error ? error.message : error}`);
2383
3317
  } finally {
2384
- await rm3(scratch, { recursive: true, force: true });
3318
+ await rm4(scratch, { recursive: true, force: true });
2385
3319
  }
2386
3320
  }
2387
3321
  }
2388
3322
 
2389
3323
  // src/local-runtime/evaluation.ts
2390
- import { createHash as createHash5 } from "crypto";
2391
- import { mkdir as mkdir6, readFile as readFile6, rm as rm4, stat as stat6, writeFile as writeFile5 } from "fs/promises";
2392
- import { dirname as dirname4, join as join5, resolve as resolve5 } from "path";
3324
+ import { createHash as createHash7 } from "crypto";
3325
+ import { mkdir as mkdir8, readFile as readFile6, rm as rm5, stat as stat7, writeFile as writeFile7 } from "fs/promises";
3326
+ import { basename as basename4, dirname as dirname6, join as join9, resolve as resolve6 } from "path";
2393
3327
  function normalize(value) {
2394
3328
  return value.trim().replace(/\s+/g, " ").toLowerCase();
2395
3329
  }
@@ -2512,8 +3446,8 @@ function scoreJsonFields(expected, actual, configuredFields) {
2512
3446
  const actualKeys = actualJson ? Object.keys(actualJson).sort() : [];
2513
3447
  const schemaMatch = Boolean(actualJson) && expectedKeys.length === actualKeys.length && expectedKeys.every((key, index) => key === actualKeys[index]);
2514
3448
  const fieldResults = {};
2515
- for (const field of fields) {
2516
- fieldResults[field] = Object.prototype.hasOwnProperty.call(expectedJson, field) && Boolean(actualJson) && JSON.stringify(canonicalJson4(expectedJson[field])) === JSON.stringify(canonicalJson4(actualJson?.[field]));
3449
+ for (const field2 of fields) {
3450
+ fieldResults[field2] = Object.prototype.hasOwnProperty.call(expectedJson, field2) && Boolean(actualJson) && JSON.stringify(canonicalJson4(expectedJson[field2])) === JSON.stringify(canonicalJson4(actualJson?.[field2]));
2517
3451
  }
2518
3452
  const correct = Object.values(fieldResults).filter(Boolean).length;
2519
3453
  const score = fields.length > 0 ? correct / fields.length : 0;
@@ -2531,10 +3465,10 @@ function aggregateJsonFieldMetrics(scores, total) {
2531
3465
  if (total === 0 || scores.length === 0) return void 0;
2532
3466
  const fields = [...new Set(scores.flatMap((score) => score.fields))].sort();
2533
3467
  const field_accuracy = {};
2534
- for (const field of fields) {
2535
- const applicable = scores.filter((score) => score.fields.includes(field));
2536
- const correct = applicable.filter((score) => score.fieldResults[field]).length;
2537
- field_accuracy[field] = {
3468
+ for (const field2 of fields) {
3469
+ const applicable = scores.filter((score) => score.fields.includes(field2));
3470
+ const correct = applicable.filter((score) => score.fieldResults[field2]).length;
3471
+ field_accuracy[field2] = {
2538
3472
  correct,
2539
3473
  total: applicable.length,
2540
3474
  accuracy: applicable.length > 0 ? correct / applicable.length : 0
@@ -2559,6 +3493,17 @@ function fileUriToPath(value) {
2559
3493
  return value.startsWith("file://") ? value.slice("file://".length) : value;
2560
3494
  }
2561
3495
  var INFERENCE_PROTOCOL_VERSION = 2;
3496
+ function parseInferenceProgress(line, total) {
3497
+ const prefix = "@@tt-evaluation ";
3498
+ if (!line.startsWith(prefix)) return null;
3499
+ try {
3500
+ const value = JSON.parse(line.slice(prefix.length));
3501
+ if (!value || !["initializing", "loading", "resuming", "generating", "reducing_batch"].includes(value.phase) || value.total !== total || !Number.isInteger(value.completed) || value.completed < 0 || value.completed > total || !Number.isInteger(value.batch_size) || value.batch_size < 1 || value.batch_size > 128 || !Number.isFinite(value.elapsed_seconds) || value.elapsed_seconds < 0 || value.eta_seconds !== null && (!Number.isFinite(value.eta_seconds) || value.eta_seconds < 0)) return null;
3502
+ return value;
3503
+ } catch {
3504
+ return null;
3505
+ }
3506
+ }
2562
3507
  function parseBatchInferenceResult(value, expectedIds) {
2563
3508
  if (!value || typeof value !== "object" || Array.isArray(value)) {
2564
3509
  throw new Error("Transformers inference output must be a JSON object");
@@ -2609,8 +3554,9 @@ async function runTransformersInference(args) {
2609
3554
  const inputPath = `${args.outputPath}.inference-input.json`;
2610
3555
  const outputPath = `${args.outputPath}.inference-output.json`;
2611
3556
  const logPath = `${args.outputPath}.inference.log`;
2612
- await mkdir6(dirname4(inputPath), { recursive: true });
2613
- await writeFile5(inputPath, `${JSON.stringify({
3557
+ const checkpointPath = args.checkpointPath ?? join9(dirname6(args.outputPath), "inference-checkpoints", basename4(args.outputPath));
3558
+ await mkdir8(dirname6(inputPath), { recursive: true });
3559
+ const payload = {
2614
3560
  protocol_version: INFERENCE_PROTOCOL_VERSION,
2615
3561
  kind: args.kind,
2616
3562
  model_id: args.modelId,
@@ -2624,25 +3570,36 @@ async function runTransformersInference(args) {
2624
3570
  id: String(index),
2625
3571
  input: example.input
2626
3572
  })),
2627
- model_cache: args.config.paths.modelCache ? resolve5(args.config.paths.modelCache) : void 0,
3573
+ model_cache: args.config.paths.modelCache ? resolve6(args.config.paths.modelCache) : void 0,
2628
3574
  trust_remote_code: false,
2629
3575
  device: args.config.evaluation.inference.device,
3576
+ batch_size: args.config.evaluation.inference.batchSize,
3577
+ source_fingerprint: args.sourceFingerprint,
2630
3578
  generation: {
2631
3579
  max_new_tokens: args.config.evaluation.inference.maxNewTokens,
2632
3580
  temperature: args.config.evaluation.inference.temperature,
2633
3581
  top_p: args.config.evaluation.inference.topP
2634
3582
  }
2635
- }, null, 2)}
3583
+ };
3584
+ const checkpointKey = baselineInputsAreStable({
3585
+ baseModelRevision: args.baseModelRevision,
3586
+ sourceFingerprint: args.sourceFingerprint,
3587
+ config: args.config
3588
+ }) ? createHash7("sha256").update(JSON.stringify({
3589
+ payload,
3590
+ package_version: await packageVersion()
3591
+ })).digest("hex") : void 0;
3592
+ await writeFile7(inputPath, `${JSON.stringify({ ...payload, checkpoint_key: checkpointKey }, null, 2)}
2636
3593
  `, "utf8");
2637
- await rm4(outputPath, { force: true });
3594
+ await rm5(outputPath, { force: true });
2638
3595
  const entrypoint = buildBundledPythonCommand(
2639
3596
  "evaluate.py",
2640
- ["--input", inputPath, "--output", outputPath]
3597
+ ["--input", inputPath, "--output", outputPath, "--checkpoint", checkpointPath]
2641
3598
  );
2642
3599
  await args.reporter?.onEvent?.({
2643
3600
  stage: `evaluating_${args.kind}`,
2644
3601
  status: "running",
2645
- message: `Starting ${args.kind} Transformers inference.`,
3602
+ message: `Starting ${args.kind} Transformers inference (batch size ${payload.batch_size}).`,
2646
3603
  details: {
2647
3604
  model_id: args.modelId,
2648
3605
  examples: args.examples.length,
@@ -2658,11 +3615,12 @@ async function runTransformersInference(args) {
2658
3615
  const files = [
2659
3616
  { path: inputPath, direction: "input" },
2660
3617
  { path: outputPath, direction: "output" },
3618
+ { path: checkpointPath, direction: "both" },
2661
3619
  { path: document.model_source, direction: "input", directory: true },
2662
3620
  ...document.adapter_path ? [{
2663
3621
  path: document.adapter_path,
2664
3622
  direction: "input",
2665
- directory: (await stat6(document.adapter_path)).isDirectory()
3623
+ directory: (await stat7(document.adapter_path)).isDirectory()
2666
3624
  }] : []
2667
3625
  ];
2668
3626
  return runGpuProcess({
@@ -2673,25 +3631,85 @@ async function runTransformersInference(args) {
2673
3631
  document: { path: inputPath, value: document, pathKeys: ["model_source", "adapter_path"] }
2674
3632
  });
2675
3633
  } : runLoggedProcess;
2676
- const result = await execute({
2677
- command: entrypoint.command,
2678
- commandArgs: entrypoint.commandArgs,
2679
- env: withBundledPythonEnvironment(
2680
- withOfflineHuggingFaceCacheEnvironment(
2681
- minimalMachineLearningEnvironment(process.env),
2682
- args.config.paths.modelCache
2683
- )
2684
- ),
2685
- logPath,
2686
- timeoutMs: args.config.evaluation.timeoutMs,
2687
- timeoutMessage: `Transformers inference timed out after ${args.config.evaluation.timeoutMs}ms`,
2688
- reporter: args.reporter,
2689
- stage: `evaluating_${args.kind}`,
2690
- shouldCancel: args.shouldCancel
2691
- });
3634
+ const started = Date.now();
3635
+ const progressState = { latest: null };
3636
+ let generatedFrom = 0;
3637
+ let lastReportedAt = 0;
3638
+ let warned = false;
3639
+ let reporting = Promise.resolve();
3640
+ const reportProgress = (progress, heartbeat2 = false) => {
3641
+ const phaseChanged = progress.phase !== progressState.latest?.phase;
3642
+ progressState.latest = progress;
3643
+ if (progress.phase === "generating" && progress.elapsed_seconds === 0) generatedFrom = progress.completed;
3644
+ const remainingSeconds = Math.max(0, (args.config.evaluation.timeoutMs - (Date.now() - started)) / 1e3);
3645
+ const overBudget = progress.eta_seconds !== null && progress.eta_seconds > remainingSeconds;
3646
+ const newWarning = overBudget && !warned;
3647
+ if (!heartbeat2 && !phaseChanged && !newWarning && progress.completed !== progress.total && Date.now() - lastReportedAt < 3e4) return;
3648
+ warned ||= overBudget;
3649
+ lastReportedAt = Date.now();
3650
+ const eta = progress.eta_seconds === null ? "" : ` ETA ${Math.ceil(progress.eta_seconds / 60)} min.`;
3651
+ const rate = progress.elapsed_seconds > 0 ? (progress.completed - generatedFrom) / progress.elapsed_seconds : null;
3652
+ const message = progress.phase === "reducing_batch" ? `CUDA memory limit reached; retrying with batch size ${progress.batch_size}.` : `${heartbeat2 ? "Still working: " : ""}${progress.phase}: ${progress.completed}/${progress.total} examples.` + (rate === null ? "" : ` ${rate.toFixed(2)} examples/s.`) + eta;
3653
+ reporting = reporting.then(async () => {
3654
+ await args.reporter?.onEvent?.({
3655
+ stage: `evaluating_${args.kind}`,
3656
+ status: "running",
3657
+ message: message + (newWarning ? " Estimated completion exceeds the timeout; increase evaluation.timeoutMs or set evaluation.maxExamples for a smoke run." : ""),
3658
+ details: {
3659
+ completed: progress.completed,
3660
+ total: progress.total,
3661
+ batch_size: progress.batch_size,
3662
+ eta_seconds: progress.eta_seconds,
3663
+ checkpoint_path: checkpointPath
3664
+ }
3665
+ });
3666
+ }).catch(() => void 0);
3667
+ };
3668
+ const heartbeat = setInterval(() => reportProgress(progressState.latest ?? {
3669
+ phase: "initializing",
3670
+ completed: 0,
3671
+ total: args.examples.length,
3672
+ batch_size: payload.batch_size,
3673
+ elapsed_seconds: 0,
3674
+ eta_seconds: null
3675
+ }, true), 3e4);
3676
+ heartbeat.unref();
3677
+ let result;
3678
+ try {
3679
+ result = await execute({
3680
+ command: entrypoint.command,
3681
+ commandArgs: entrypoint.commandArgs,
3682
+ env: withBundledPythonEnvironment(
3683
+ withOfflineHuggingFaceCacheEnvironment(
3684
+ minimalMachineLearningEnvironment(process.env),
3685
+ args.config.paths.modelCache
3686
+ )
3687
+ ),
3688
+ logPath,
3689
+ timeoutMs: args.config.evaluation.timeoutMs,
3690
+ timeoutMessage: `Transformers inference timed out after ${args.config.evaluation.timeoutMs}ms`,
3691
+ reporter: args.reporter,
3692
+ stage: `evaluating_${args.kind}`,
3693
+ shouldCancel: args.shouldCancel,
3694
+ onLine: (line, stream) => {
3695
+ if (stream !== "stdout") return;
3696
+ const progress = parseInferenceProgress(line, args.examples.length);
3697
+ if (progress) reportProgress(progress);
3698
+ }
3699
+ });
3700
+ } catch (error) {
3701
+ if (error instanceof Error && error.message.startsWith("Transformers inference timed out")) {
3702
+ throw new Error(`${error.message}. Completed ${progressState.latest?.completed ?? 0}/${args.examples.length} examples. Checkpoint: ${checkpointPath}. Retry with the same inputs and a larger evaluation.timeoutMs.`);
3703
+ }
3704
+ throw error;
3705
+ } finally {
3706
+ clearInterval(heartbeat);
3707
+ await reporting;
3708
+ }
2692
3709
  if (result.exitCode !== 0) {
2693
3710
  throw new Error(
2694
- `Transformers inference exited ${result.exitCode}: ${result.stderr.slice(0, 1e3)}`
3711
+ `Transformers inference exited ${result.exitCode}: ${result.stderr.slice(-2e3)}
3712
+ Checkpoint: ${checkpointPath}. Full log: ${logPath}`
2695
3713
  );
2696
3714
  }
2697
3715
  let output;
@@ -2708,13 +3726,13 @@ async function runTransformersInference(args) {
2708
3726
  );
2709
3727
  }
2710
3728
  function assertEvaluationScoringReady(config) {
2711
- if (config.evaluation.scoring.mode === "json_fields" && config.evaluation.scoring.fields?.some((field) => !field.trim())) {
3729
+ if (config.evaluation.scoring.mode === "json_fields" && config.evaluation.scoring.fields?.some((field2) => !field2.trim())) {
2712
3730
  throw new Error("evaluation.scoring.fields must contain non-empty field names");
2713
3731
  }
2714
3732
  }
2715
3733
  function baselineCacheKey(args) {
2716
- return createHash5("sha256").update(JSON.stringify({
2717
- v: 6,
3734
+ return createHash7("sha256").update(JSON.stringify({
3735
+ v: 7,
2718
3736
  inference_protocol_version: INFERENCE_PROTOCOL_VERSION,
2719
3737
  package_version: args.packageVersion,
2720
3738
  model_id: args.modelId,
@@ -2733,10 +3751,10 @@ function baselineInputsAreStable(args) {
2733
3751
  return args.config.paths.baseModel ? Boolean(args.sourceFingerprint) : Boolean(args.baseModelRevision);
2734
3752
  }
2735
3753
  function baselineStoreRoot(config) {
2736
- return config.storeRoot ? resolve5(config.storeRoot) : defaultLocalHome();
3754
+ return config.storeRoot ? resolve6(config.storeRoot) : defaultLocalHome();
2737
3755
  }
2738
3756
  function baselineCachePath(config, key) {
2739
- return join5(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
3757
+ return join9(baselineStoreRoot(config), "cache", "baseline-evals", `${key}.json`);
2740
3758
  }
2741
3759
  async function readBaselineCache(config, path) {
2742
3760
  await hardenExistingLocalStore(baselineStoreRoot(config));
@@ -2750,7 +3768,7 @@ var cachedPackageVersion = null;
2750
3768
  async function packageVersion() {
2751
3769
  if (cachedPackageVersion !== null) return cachedPackageVersion;
2752
3770
  try {
2753
- const raw = await readFile6(resolve5(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
3771
+ const raw = await readFile6(resolve6(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
2754
3772
  cachedPackageVersion = String(
2755
3773
  JSON.parse(raw).version ?? "unknown"
2756
3774
  );
@@ -2819,7 +3837,7 @@ async function evaluateExamples(args) {
2819
3837
  baseModelId,
2820
3838
  args.baseModelRevision
2821
3839
  );
2822
- const localModelSource = args.modelSource ? resolve5(args.modelSource) !== resolve5(baseModelId) : false;
3840
+ const localModelSource = args.modelSource ? resolve6(args.modelSource) !== resolve6(baseModelId) : false;
2823
3841
  const baseModelRevision = localModelSource ? defaultBaseModelRevision(baseModelId) : requestedBaseModelRevision;
2824
3842
  const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
2825
3843
  const truncated = args.examples.length > maxExamples;
@@ -2861,6 +3879,11 @@ async function evaluateExamples(args) {
2861
3879
  return report2;
2862
3880
  }
2863
3881
  }
3882
+ if (cacheKey) {
3883
+ await createLocalStore(baselineStoreRoot(args.config)).ensure();
3884
+ await mkdir8(dirname6(baselineCachePath(args.config, cacheKey)), { recursive: true, mode: 448 });
3885
+ await hardenExistingLocalStore(baselineStoreRoot(args.config));
3886
+ }
2864
3887
  const inference = args.config.dryRun ? null : await runTransformersInference({
2865
3888
  kind: args.kind,
2866
3889
  modelId: args.modelId,
@@ -2868,6 +3891,8 @@ async function evaluateExamples(args) {
2868
3891
  modelSource: args.modelSource ?? args.modelId,
2869
3892
  baseModelRevision,
2870
3893
  adapterPath: args.adapterPath,
3894
+ sourceFingerprint: args.sourceFingerprint,
3895
+ ...cacheKey ? { checkpointPath: `${baselineCachePath(args.config, cacheKey)}.checkpoint.json` } : {},
2871
3896
  examples,
2872
3897
  system: args.system,
2873
3898
  config: args.config,
@@ -2935,8 +3960,8 @@ function compareEvalReports(baseline, candidate) {
2935
3960
  }
2936
3961
 
2937
3962
  // src/local-runtime/process-training.ts
2938
- import { access, mkdir as mkdir7, readFile as readFile7, writeFile as writeFile6 } from "fs/promises";
2939
- import { join as join6, resolve as resolve6 } from "path";
3963
+ import { access, mkdir as mkdir9, readFile as readFile7, writeFile as writeFile8 } from "fs/promises";
3964
+ import { join as join10, resolve as resolve7 } from "path";
2940
3965
  function buildTrainingHyperparameters(request, options = {}) {
2941
3966
  const model = resolveTrainingModel(request.spec_snapshot.base_model);
2942
3967
  const hyper = request.hyperparameters;
@@ -3027,15 +4052,15 @@ function createTrainingProgressForwarder(reporter) {
3027
4052
  async function launchProcessTraining(args) {
3028
4053
  const { request, artifacts, config } = args;
3029
4054
  const jobName = `tt-local-${request.run_id}`;
3030
- await mkdir7(artifacts.trainingDir, { recursive: true });
4055
+ await mkdir9(artifacts.trainingDir, { recursive: true });
3031
4056
  await copyDatasetToTrainingChannel(artifacts);
3032
4057
  const hyperparameters = buildTrainingHyperparameters(request, {
3033
4058
  baseModelRevision: args.baseModelRevision
3034
4059
  });
3035
- await writeJson(join6(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
4060
+ await writeJson(join10(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
3036
4061
  const entrypoint = buildBundledPythonCommand("train.py");
3037
4062
  if (config.dryRun) {
3038
- await writeFile6(
4063
+ await writeFile8(
3039
4064
  artifacts.trainingLog,
3040
4065
  "Dry run enabled. Training process was not launched.\n",
3041
4066
  "utf8"
@@ -3057,11 +4082,11 @@ async function launchProcessTraining(args) {
3057
4082
  withOfflineHuggingFaceCacheEnvironment({
3058
4083
  ...inheritedEnv,
3059
4084
  BACKEND: "local",
3060
- SM_CHANNEL_TRAINING: resolve6(artifacts.trainingInputDir),
3061
- SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve6(config.paths.baseModel) : void 0,
3062
- SM_MODEL_DIR: resolve6(artifacts.trainingModelDir),
3063
- SM_OUTPUT_DIR: resolve6(artifacts.trainingOutputDir),
3064
- TT_HYPERPARAMETERS_PATH: resolve6(artifacts.trainingConfigDir, "hyperparameters.json")
4085
+ SM_CHANNEL_TRAINING: resolve7(artifacts.trainingInputDir),
4086
+ SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve7(config.paths.baseModel) : void 0,
4087
+ SM_MODEL_DIR: resolve7(artifacts.trainingModelDir),
4088
+ SM_OUTPUT_DIR: resolve7(artifacts.trainingOutputDir),
4089
+ TT_HYPERPARAMETERS_PATH: resolve7(artifacts.trainingConfigDir, "hyperparameters.json")
3065
4090
  }, config.paths.modelCache)
3066
4091
  );
3067
4092
  for (const [key, value] of Object.entries(childEnv)) {
@@ -3069,10 +4094,10 @@ async function launchProcessTraining(args) {
3069
4094
  }
3070
4095
  const remoteBaseModel = config.gpu ? gpuBaseModelPath(config, request.spec_snapshot.base_model, args.baseModelRevision) : void 0;
3071
4096
  if (remoteBaseModel) childEnv.SM_CHANNEL_BASE_MODEL = remoteBaseModel;
3072
- if (config.paths.modelCache) await mkdir7(resolve6(config.paths.modelCache), { recursive: true });
4097
+ if (config.paths.modelCache) await mkdir9(resolve7(config.paths.modelCache), { recursive: true });
3073
4098
  if (config.paths.baseModel) {
3074
4099
  await verifyLocalBaseModel(
3075
- resolve6(config.paths.baseModel),
4100
+ resolve7(config.paths.baseModel),
3076
4101
  request.spec_snapshot.base_model,
3077
4102
  config.paths.modelCache
3078
4103
  );
@@ -3111,12 +4136,12 @@ async function launchProcessTraining(args) {
3111
4136
  onLine: (line) => forwardTrainingProgress(line),
3112
4137
  shouldCancel: args.shouldCancel
3113
4138
  });
3114
- const metricsPath = join6(artifacts.trainingModelDir, "training-metrics.json");
4139
+ const metricsPath = join10(artifacts.trainingModelDir, "training-metrics.json");
3115
4140
  const metrics = await pathExists(metricsPath) ? JSON.parse(await readFile7(metricsPath, "utf8")) : null;
3116
4141
  if (exitCode !== 0) {
3117
4142
  throw new Error(`Training process exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
3118
4143
  }
3119
- const modelTarPath = join6(artifacts.trainingOutputDir, "model.tar.gz");
4144
+ const modelTarPath = join10(artifacts.trainingOutputDir, "model.tar.gz");
3120
4145
  const modelUri = await pathExists(modelTarPath) ? fileUri(modelTarPath) : fileUri(artifacts.trainingModelDir);
3121
4146
  await args.reporter?.onEvent?.({
3122
4147
  stage: "training",
@@ -3196,10 +4221,10 @@ function validateLocalPipeline(pipeline) {
3196
4221
  }
3197
4222
  }
3198
4223
  if (step.uses === "compare") {
3199
- for (const field of ["before", "after"]) {
3200
- const reportReference = referenceParts(step.with?.[field]);
4224
+ for (const field2 of ["before", "after"]) {
4225
+ const reportReference = referenceParts(step.with?.[field2]);
3201
4226
  if (!reportReference || reportReference[1] !== "report") {
3202
- throw new Error(`Local compare step ${step.id} requires ${field}: { from: '<earlier-evaluate>.report' }.`);
4227
+ throw new Error(`Local compare step ${step.id} requires ${field2}: { from: '<earlier-evaluate>.report' }.`);
3203
4228
  }
3204
4229
  const producer = seen.get(reportReference[0]);
3205
4230
  if (!producer) throw new Error(`Local compare step ${step.id} must reference an earlier evaluate step.`);
@@ -3219,13 +4244,13 @@ async function loadRunRequest(path) {
3219
4244
  function parseLocalRunnerConfig(document, path) {
3220
4245
  const config = localRunnerConfigSchema.parse(document);
3221
4246
  if (!path) return config;
3222
- const configPath = resolve7(path);
3223
- const base = dirname5(configPath);
4247
+ const configPath = resolve8(path);
4248
+ const base = dirname7(configPath);
3224
4249
  const configPathValue = (value) => {
3225
4250
  if (!value) return void 0;
3226
4251
  if (value === "~") return homedir2();
3227
- if (value.startsWith("~/")) return resolve7(homedir2(), value.slice(2));
3228
- return resolve7(base, value);
4252
+ if (value.startsWith("~/")) return resolve8(homedir2(), value.slice(2));
4253
+ return resolve8(base, value);
3229
4254
  };
3230
4255
  return {
3231
4256
  ...config,
@@ -3247,7 +4272,7 @@ function parseLocalRunnerConfig(document, path) {
3247
4272
  }
3248
4273
  async function loadLocalRunnerConfig(path) {
3249
4274
  if (!path) return parseLocalRunnerConfig({});
3250
- const configPath = resolve7(path);
4275
+ const configPath = resolve8(path);
3251
4276
  return parseLocalRunnerConfig(
3252
4277
  await loadJsonFile(configPath),
3253
4278
  configPath
@@ -3262,7 +4287,7 @@ function stripFileUri(path) {
3262
4287
  }
3263
4288
  async function pathExists2(path) {
3264
4289
  try {
3265
- await stat7(path);
4290
+ await stat8(path);
3266
4291
  return true;
3267
4292
  } catch {
3268
4293
  return false;
@@ -3274,17 +4299,18 @@ function selectPrebuiltEvaluationSplit(dataset) {
3274
4299
  if (dataset.test) return "prebuilt_test";
3275
4300
  throw new Error("dataset_prebuilt requires a distinct validation or test split");
3276
4301
  }
3277
- function assertMatchingDatasetSystems(jsonl, expectedSystem, split) {
3278
- for (const [index, line] of jsonl.split(/\r?\n/).entries()) {
3279
- if (!line.trim()) continue;
4302
+ function applyDatasetSystem(jsonl, expectedSystem, split) {
4303
+ return jsonl.split("\n").map((line, index) => {
3280
4304
  const row = JSON.parse(line);
3281
- const actualSystem = row.messages.find((message) => message.role === "system")?.content ?? "";
3282
- if (actualSystem !== expectedSystem) {
4305
+ const system = row.messages.find((message) => message.role === "system");
4306
+ if (system && system.content !== expectedSystem) {
3283
4307
  throw new Error(
3284
4308
  `Prebuilt ${split} row ${index + 1} system message must match the behavior spec system message.`
3285
4309
  );
3286
4310
  }
3287
- }
4311
+ if (!system) row.messages.unshift({ role: "system", content: expectedSystem });
4312
+ return JSON.stringify(row);
4313
+ }).join("\n");
3288
4314
  }
3289
4315
  function addDryRunPlaceholders(request, dryRun) {
3290
4316
  if (!dryRun || !request || typeof request !== "object" || Array.isArray(request)) return request;
@@ -3348,8 +4374,7 @@ async function validateDatasetInputs(request, config) {
3348
4374
  for (const [name, value] of splitEntries) {
3349
4375
  if (!value) continue;
3350
4376
  const path = stripFileUri(value);
3351
- const jsonl = await normalizeChatJsonlForRelocation(path);
3352
- assertMatchingDatasetSystems(jsonl, expectedSystem, name);
4377
+ const jsonl = applyDatasetSystem(await normalizeChatJsonlForRelocation(path), expectedSystem, name);
3353
4378
  normalized.set(name, {
3354
4379
  jsonl,
3355
4380
  examples: await examplesFromChatJsonl(path)
@@ -3419,11 +4444,11 @@ function artifactPrefix(request) {
3419
4444
  function canonicalJson5(value) {
3420
4445
  if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
3421
4446
  if (Array.isArray(value)) return `[${value.map((entry) => canonicalJson5(entry)).join(",")}]`;
3422
- const object = value;
3423
- return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson5(object[key])}`).join(",")}}`;
4447
+ const object2 = value;
4448
+ return `{${Object.keys(object2).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson5(object2[key])}`).join(",")}}`;
3424
4449
  }
3425
4450
  function hashJson(value) {
3426
- return createHash6("sha256").update(canonicalJson5(value)).digest("hex");
4451
+ return createHash8("sha256").update(canonicalJson5(value)).digest("hex");
3427
4452
  }
3428
4453
  var packageRoot = localRuntimePackageRoot(import.meta.url);
3429
4454
  async function hashFileIfPresent(path) {
@@ -3435,21 +4460,21 @@ async function hashFileIfPresent(path) {
3435
4460
  }
3436
4461
  async function packageVersion2() {
3437
4462
  try {
3438
- const parsed = JSON.parse(await readFile8(resolve7(packageRoot, "package.json"), "utf8"));
4463
+ const parsed = JSON.parse(await readFile8(resolve8(packageRoot, "package.json"), "utf8"));
3439
4464
  return typeof parsed.version === "string" ? parsed.version : "unknown";
3440
4465
  } catch {
3441
4466
  return "unknown";
3442
4467
  }
3443
4468
  }
3444
4469
  async function runtimeFingerprint() {
3445
- const bundledProject = resolve7(packageRoot, "training/adapter");
4470
+ const bundledProject = resolve8(packageRoot, "training/adapter");
3446
4471
  return hashJson({
3447
4472
  tt_local_version: await packageVersion2(),
3448
4473
  node_version: process.version,
3449
4474
  platform: process.platform,
3450
4475
  architecture: process.arch,
3451
- runner_pyproject: await hashFileIfPresent(resolve7(bundledProject, "pyproject.toml")),
3452
- runner_uv_lock: await hashFileIfPresent(resolve7(bundledProject, "uv.lock"))
4476
+ runner_pyproject: await hashFileIfPresent(resolve8(bundledProject, "pyproject.toml")),
4477
+ runner_uv_lock: await hashFileIfPresent(resolve8(bundledProject, "uv.lock"))
3453
4478
  });
3454
4479
  }
3455
4480
  function preparedSourceFingerprint(args) {
@@ -3463,7 +4488,7 @@ function preparedSourceFingerprint(args) {
3463
4488
  });
3464
4489
  }
3465
4490
  async function hashFile(path) {
3466
- const hash = createHash6("sha256");
4491
+ const hash = createHash8("sha256");
3467
4492
  for await (const chunk of createReadStream3(path)) hash.update(chunk);
3468
4493
  return hash.digest("hex");
3469
4494
  }
@@ -3496,7 +4521,7 @@ async function resolveBaseModelRevision(request, config) {
3496
4521
  }
3497
4522
  const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
3498
4523
  const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
3499
- const refPath = resolve7(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
4524
+ const refPath = resolve8(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
3500
4525
  try {
3501
4526
  const revision = (await readFile8(refPath, "utf8")).trim();
3502
4527
  if (revision) return revision;
@@ -3506,7 +4531,7 @@ async function resolveBaseModelRevision(request, config) {
3506
4531
  }
3507
4532
  async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
3508
4533
  const root = localModelArtifactPath(uri);
3509
- const rootMetadata = await lstat2(root);
4534
+ const rootMetadata = await lstat5(root);
3510
4535
  if (rootMetadata.isSymbolicLink()) {
3511
4536
  throw new Error(`Local base model path must not itself be a symbolic link: ${root}`);
3512
4537
  }
@@ -3516,19 +4541,19 @@ async function fingerprintLocalBaseModel(uri, expectedModelId, modelCache) {
3516
4541
  await verifyLocalBaseModel(root, expectedModelId, modelCache);
3517
4542
  const files = [];
3518
4543
  const visit = async (directory) => {
3519
- for (const entry of await readdir2(directory, { withFileTypes: true })) {
3520
- const child = resolve7(directory, entry.name);
4544
+ for (const entry of await readdir3(directory, { withFileTypes: true })) {
4545
+ const child = resolve8(directory, entry.name);
3521
4546
  if (entry.isDirectory()) {
3522
4547
  await visit(child);
3523
4548
  continue;
3524
4549
  }
3525
- const childMetadata = await stat7(child);
4550
+ const childMetadata = await stat8(child);
3526
4551
  if (entry.isSymbolicLink() && !childMetadata.isFile()) {
3527
4552
  throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
3528
4553
  }
3529
4554
  if (!childMetadata.isFile()) continue;
3530
4555
  files.push({
3531
- path: relative3(root, child).split("\\").join("/"),
4556
+ path: relative6(root, child).split("\\").join("/"),
3532
4557
  size_bytes: childMetadata.size,
3533
4558
  sha256: await hashFile(child)
3534
4559
  });
@@ -3556,27 +4581,27 @@ async function readStageMetadata(path) {
3556
4581
  }
3557
4582
  }
3558
4583
  async function removePrefixedArtifacts(path) {
3559
- const directory = dirname5(path);
3560
- const prefix = `${basename3(path)}.`;
3561
- const names = await readdir2(directory).catch(() => []);
4584
+ const directory = dirname7(path);
4585
+ const prefix = `${basename5(path)}.`;
4586
+ const names = await readdir3(directory).catch(() => []);
3562
4587
  await Promise.all([
3563
- rm5(path, { force: true }),
3564
- ...names.filter((name) => name.startsWith(prefix)).map((name) => rm5(resolve7(directory, name), { recursive: true, force: true }))
4588
+ rm6(path, { force: true }),
4589
+ ...names.filter((name) => name.startsWith(prefix)).map((name) => rm6(resolve8(directory, name), { recursive: true, force: true }))
3565
4590
  ]);
3566
4591
  }
3567
4592
  async function cleanupStageArtifacts(artifacts, stage) {
3568
- const removeReport = async () => rm5(artifacts.runReportJson, { force: true });
4593
+ const removeReport = async () => rm6(artifacts.runReportJson, { force: true });
3569
4594
  if (stage === "prepare") {
3570
4595
  await Promise.all([
3571
4596
  removePrefixedArtifacts(artifacts.baselineEvalJson),
3572
4597
  removePrefixedArtifacts(artifacts.candidateEvalJson),
3573
4598
  removePrefixedArtifacts(artifacts.generalBaselineEvalJson),
3574
4599
  removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
3575
- rm5(artifacts.trainingDir, { recursive: true, force: true }),
4600
+ rm6(artifacts.trainingDir, { recursive: true, force: true }),
3576
4601
  removePrefixedArtifacts(artifacts.trainingReportJson),
3577
- rm5(resolve7(artifacts.runDir, "model.tar.gz"), { force: true }),
4602
+ rm6(resolve8(artifacts.runDir, "model.tar.gz"), { force: true }),
3578
4603
  removeReport(),
3579
- rm5(artifacts.artifactManifestJson, { force: true })
4604
+ rm6(artifacts.artifactManifestJson, { force: true })
3580
4605
  ]);
3581
4606
  return;
3582
4607
  }
@@ -3590,9 +4615,9 @@ async function cleanupStageArtifacts(artifacts, stage) {
3590
4615
  }
3591
4616
  if (stage === "train") {
3592
4617
  await Promise.all([
3593
- rm5(artifacts.trainingDir, { recursive: true, force: true }),
4618
+ rm6(artifacts.trainingDir, { recursive: true, force: true }),
3594
4619
  removePrefixedArtifacts(artifacts.trainingReportJson),
3595
- rm5(resolve7(artifacts.runDir, "model.tar.gz"), { force: true }),
4620
+ rm6(resolve8(artifacts.runDir, "model.tar.gz"), { force: true }),
3596
4621
  removePrefixedArtifacts(artifacts.candidateEvalJson),
3597
4622
  removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
3598
4623
  removeReport()
@@ -3611,7 +4636,7 @@ async function cleanupStageArtifacts(artifacts, stage) {
3611
4636
  await removeReport();
3612
4637
  }
3613
4638
  function manifestRelativePath(artifacts, path) {
3614
- return relative3(artifacts.runDir, path).split("\\").join("/");
4639
+ return relative6(artifacts.runDir, path).split("\\").join("/");
3615
4640
  }
3616
4641
  async function verifyReusableArtifacts(artifacts, paths, options = {}) {
3617
4642
  if (!await pathExists2(artifacts.artifactManifestJson)) return false;
@@ -3628,11 +4653,11 @@ async function throwIfCancelled(store, request) {
3628
4653
  }
3629
4654
  }
3630
4655
  async function acquireWorkflowLock(lockPath, description) {
3631
- await mkdir8(dirname5(lockPath), { recursive: true });
3632
- const token = randomUUID5();
4656
+ await mkdir10(dirname7(lockPath), { recursive: true });
4657
+ const token = randomUUID8();
3633
4658
  for (let attempt = 0; attempt < 3; attempt += 1) {
3634
4659
  try {
3635
- const handle = await open2(lockPath, "wx", 384);
4660
+ const handle = await open6(lockPath, "wx", 384);
3636
4661
  try {
3637
4662
  await handle.writeFile(`${JSON.stringify({ token, pid: process.pid, created_at: (/* @__PURE__ */ new Date()).toISOString() })}
3638
4663
  `);
@@ -3642,13 +4667,13 @@ async function acquireWorkflowLock(lockPath, description) {
3642
4667
  return async () => {
3643
4668
  try {
3644
4669
  const owner = JSON.parse(await readFile8(lockPath, "utf8"));
3645
- if (owner.token === token) await rm5(lockPath, { force: true });
4670
+ if (owner.token === token) await rm6(lockPath, { force: true });
3646
4671
  } catch {
3647
4672
  }
3648
4673
  };
3649
4674
  } catch (error) {
3650
4675
  if (error.code !== "EEXIST") throw error;
3651
- const metadata = await stat7(lockPath).catch(() => null);
4676
+ const metadata = await stat8(lockPath).catch(() => null);
3652
4677
  let owner = null;
3653
4678
  try {
3654
4679
  owner = JSON.parse(await readFile8(lockPath, "utf8"));
@@ -3670,10 +4695,10 @@ async function acquireWorkflowLock(lockPath, description) {
3670
4695
  `${description} already has an active local workflow (lock: ${lockPath}).`
3671
4696
  );
3672
4697
  }
3673
- const stalePath = `${lockPath}.stale.${randomUUID5()}`;
4698
+ const stalePath = `${lockPath}.stale.${randomUUID8()}`;
3674
4699
  try {
3675
- await rename3(lockPath, stalePath);
3676
- await rm5(stalePath, { force: true });
4700
+ await rename5(lockPath, stalePath);
4701
+ await rm6(stalePath, { force: true });
3677
4702
  } catch (renameError) {
3678
4703
  if (renameError.code !== "ENOENT") throw renameError;
3679
4704
  }
@@ -3683,11 +4708,11 @@ async function acquireWorkflowLock(lockPath, description) {
3683
4708
  }
3684
4709
  async function acquireRunLock(store, runId) {
3685
4710
  await store.ensure();
3686
- return acquireWorkflowLock(resolve7(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
4711
+ return acquireWorkflowLock(resolve8(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
3687
4712
  }
3688
4713
  async function acquireArtifactLock(artifacts) {
3689
4714
  return acquireWorkflowLock(
3690
- join7(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
4715
+ join11(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
3691
4716
  `Artifact directory ${artifacts.runDir}`
3692
4717
  );
3693
4718
  }
@@ -3734,13 +4759,13 @@ function stageFingerprintPath(prepared, stage) {
3734
4759
  return `${stageOutputPath(prepared, stage)}.stage.json`;
3735
4760
  }
3736
4761
  async function runnerFileFingerprints() {
3737
- const project = resolve7(packageRoot, "training/adapter");
3738
- const bundledSource = resolve7(packageRoot, "training/adapter/src");
3739
- const pythonSources = (await readdir2(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve7(bundledSource, entry.name));
4762
+ const project = resolve8(packageRoot, "training/adapter");
4763
+ const bundledSource = resolve8(packageRoot, "training/adapter/src");
4764
+ const pythonSources = (await readdir3(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve8(bundledSource, entry.name));
3740
4765
  const candidates = [.../* @__PURE__ */ new Set([
3741
4766
  ...pythonSources,
3742
- resolve7(project, "pyproject.toml"),
3743
- resolve7(project, "uv.lock")
4767
+ resolve8(project, "pyproject.toml"),
4768
+ resolve8(project, "uv.lock")
3744
4769
  ])];
3745
4770
  const fingerprints = {};
3746
4771
  for (const path of candidates.sort()) fingerprints[path] = await hashFileIfPresent(path);
@@ -3854,7 +4879,7 @@ async function ensureRunRecord(args) {
3854
4879
  `Run ${args.request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
3855
4880
  );
3856
4881
  }
3857
- if (resolve7(existing.artifact_dir) !== resolve7(args.artifacts.runDir)) {
4882
+ if (resolve8(existing.artifact_dir) !== resolve8(args.artifacts.runDir)) {
3858
4883
  throw new Error(
3859
4884
  `Run ${args.request.run_id} is already bound to artifact directory ${existing.artifact_dir}; it cannot be resumed with ${args.artifacts.runDir}.`
3860
4885
  );
@@ -3932,7 +4957,7 @@ async function computePreparedRun(args) {
3932
4957
  base_model_for_evaluation: baseModelForEvaluation,
3933
4958
  base_model_revision: baseModelRevision ?? null,
3934
4959
  base_model_fingerprint: baseModelFingerprint ?? null,
3935
- system_prompt_sha256: createHash6("sha256").update(system).digest("hex"),
4960
+ system_prompt_sha256: createHash8("sha256").update(system).digest("hex"),
3936
4961
  prepared_at: (/* @__PURE__ */ new Date()).toISOString()
3937
4962
  };
3938
4963
  if (args.writeArtifacts) await writeJsonAtomic(artifacts.stageMetadataJson, metadata);
@@ -3961,7 +4986,7 @@ async function prepareStage(args) {
3961
4986
  const preparedExists = await pathExists2(args.artifacts.stageMetadataJson) && await pathExists2(args.artifacts.trainingJsonl);
3962
4987
  const prepared = await computePreparedRun({ ...args, writeArtifacts: false });
3963
4988
  await args.store.syncRunRequest(args.request, args.artifacts.runDir);
3964
- await writeJsonAtomic(resolve7(args.artifacts.runDir, "request.json"), args.request);
4989
+ await writeJsonAtomic(resolve8(args.artifacts.runDir, "request.json"), args.request);
3965
4990
  const existingMetadata = preparedExists ? await readStageMetadata(args.artifacts.stageMetadataJson) : null;
3966
4991
  let canReuse = preparedExists && existingMetadata?.source_fingerprint === prepared.metadata.source_fingerprint;
3967
4992
  if (canReuse) {
@@ -4247,6 +5272,7 @@ async function runCandidateStage(args) {
4247
5272
  modelSource: args.prepared.baseModelForEvaluation,
4248
5273
  baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
4249
5274
  adapterPath: modelArtifact,
5275
+ sourceFingerprint: args.prepared.metadata.base_model_fingerprint ?? void 0,
4250
5276
  examples: args.prepared.examples,
4251
5277
  system: args.prepared.system,
4252
5278
  config: args.config,
@@ -4277,6 +5303,7 @@ async function runCandidateStage(args) {
4277
5303
  modelSource: args.prepared.baseModelForEvaluation,
4278
5304
  baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
4279
5305
  adapterPath: modelArtifact,
5306
+ sourceFingerprint: args.prepared.metadata.base_model_fingerprint ?? void 0,
4280
5307
  examples: args.prepared.generalRegression.examples,
4281
5308
  system: args.prepared.generalRegression.system,
4282
5309
  config: generalRegressionEvaluationConfig(args.config),
@@ -4307,7 +5334,7 @@ async function buildRunProvenance(args) {
4307
5334
  platform: process.platform,
4308
5335
  arch: process.arch,
4309
5336
  runtime_fingerprint: metadata.runtime_fingerprint,
4310
- python_lock_sha256: await hashFileIfPresent(resolve7(packageRoot, "training/adapter/uv.lock"))
5337
+ python_lock_sha256: await hashFileIfPresent(resolve8(packageRoot, "training/adapter/uv.lock"))
4311
5338
  },
4312
5339
  execution: {
4313
5340
  target: gpu ? "aws" : "local",
@@ -4545,7 +5572,7 @@ async function runLocalPipeline(input) {
4545
5572
  store,
4546
5573
  reporter: input.reporter
4547
5574
  });
4548
- await writeJsonAtomic(join7(artifacts.runDir, "resolved-workflow.json"), {
5575
+ await writeJsonAtomic(join11(artifacts.runDir, "resolved-workflow.json"), {
4549
5576
  spec: input.projectSpec ?? {
4550
5577
  ...input.request.spec_snapshot,
4551
5578
  hyperparameters: input.request.hyperparameters,
@@ -4567,7 +5594,7 @@ async function runLocalPipeline(input) {
4567
5594
  if (step.uses === "evaluate") {
4568
5595
  const isBase = step.with.model === "base";
4569
5596
  const report = isBase ? await runBaselineStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter }) : await runCandidateStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
4570
- const reportPath = join7(artifacts.runDir, "evaluations", `${step.id}.json`);
5597
+ const reportPath = join11(artifacts.runDir, "evaluations", `${step.id}.json`);
4571
5598
  await writeJsonAtomic(reportPath, { ...report, artifact_uri: fileUri(reportPath) });
4572
5599
  outputs[step.id] = { report: { ...report, artifact_uri: fileUri(reportPath) }, reportPath };
4573
5600
  await refreshArtifactManifest(prepared);
@@ -4646,7 +5673,7 @@ async function runLocalFineTune(input) {
4646
5673
  // src/local-runtime/project-workflow.ts
4647
5674
  import { existsSync } from "fs";
4648
5675
  import { readFile as readFile9 } from "fs/promises";
4649
- import { basename as basename4, dirname as dirname6, join as join8, resolve as resolve8 } from "path";
5676
+ import { basename as basename6, dirname as dirname8, join as join12, resolve as resolve9 } from "path";
4650
5677
  function configForSpec(spec, specPath) {
4651
5678
  return parseLocalRunnerConfig({
4652
5679
  storeRoot: process.env.TT_LOCAL_HOME?.trim() ? expandUserPath(process.env.TT_LOCAL_HOME.trim()) : DEFAULT_PROJECT_STORE_ROOT,
@@ -4662,15 +5689,15 @@ async function resolveDownloadConfig(specPath, configPath, warn) {
4662
5689
  if (error.code) throw error;
4663
5690
  }
4664
5691
  if (input?.kind === "spec") return (await resolveProjectConfig(specPath, configPath, warn, input)).config;
4665
- return configPath ? parseLocalRunnerConfig(JSON.parse(await readFile9(configPath, "utf8")), resolve8(configPath)) : parseLocalRunnerConfig({});
5692
+ return configPath ? parseLocalRunnerConfig(JSON.parse(await readFile9(configPath, "utf8")), resolve9(configPath)) : parseLocalRunnerConfig({});
4666
5693
  }
4667
5694
  async function resolveProjectConfig(specPath, configPath, warn, loaded) {
4668
5695
  const input = loaded ?? (existsSync(specPath) ? await loadLocalRunInput(specPath) : void 0);
4669
5696
  const spec = input && input.kind !== "request" ? input.spec : void 0;
4670
- if (!configPath && spec?.runtime && /^\..+\.tt-agent-[0-9a-f-]+\.json$/.test(basename4(specPath))) {
5697
+ if (!configPath && spec?.runtime && /^\..+\.tt-agent-[0-9a-f-]+\.json$/.test(basename6(specPath))) {
4671
5698
  return { config: configForSpec(spec, specPath) };
4672
5699
  }
4673
- const candidate = configPath ? resolve8(configPath) : join8(dirname6(resolve8(specPath)), "local-runner.json");
5700
+ const candidate = configPath ? resolve9(configPath) : join12(dirname8(resolve9(specPath)), "local-runner.json");
4674
5701
  if (configPath || existsSync(candidate)) {
4675
5702
  const legacy = parseLocalRunnerConfig(JSON.parse(await readFile9(candidate, "utf8")), candidate);
4676
5703
  if (spec?.runtime || spec?.evaluation) {
@@ -4682,314 +5709,219 @@ async function resolveProjectConfig(specPath, configPath, warn, loaded) {
4682
5709
  return { config: spec ? configForSpec(spec, specPath) : parseLocalRunnerConfig({}) };
4683
5710
  }
4684
5711
 
4685
- // src/local-runtime/capability-profiles.ts
4686
- var GIB = 1024 ** 3;
4687
- function gib(n) {
4688
- return Math.round(n * GIB);
5712
+ // src/local-runtime/host-inventory.ts
5713
+ import { constants as constants5 } from "fs";
5714
+ import { access as access2, statfs as statfs2 } from "fs/promises";
5715
+ import { cpus as cpus2, homedir as homedir4, type } from "os";
5716
+ import { dirname as dirname10, join as join14, resolve as resolve11 } from "path";
5717
+ import { spawn } from "child_process";
5718
+
5719
+ // src/local-runtime/live-usage.ts
5720
+ import { execFile } from "child_process";
5721
+ import { readFileSync } from "fs";
5722
+ import { statfs } from "fs/promises";
5723
+ import { cpus, freemem, homedir as homedir3, loadavg, totalmem } from "os";
5724
+ import { dirname as dirname9, join as join13, resolve as resolve10 } from "path";
5725
+ function parseMacMemory(stdout, total) {
5726
+ const pageSize = Number(/page size of (\d+) bytes/.exec(stdout)?.[1]);
5727
+ const pages = (label) => {
5728
+ const line = stdout.split(/\r?\n/).find((line2) => line2.startsWith(`${label}:`));
5729
+ const count = line && /^\s*(\d+)\.?\s*$/.exec(line.slice(label.length + 1));
5730
+ return count ? Number(count[1]) : NaN;
5731
+ };
5732
+ const free = pages("Pages free") * pageSize;
5733
+ const cached = (pages("File-backed pages") + pages("Pages purgeable")) * pageSize;
5734
+ const available = free + cached;
5735
+ if (!(pageSize > 0) || !(total > 0) || !Number.isSafeInteger(available) || available > total) return void 0;
5736
+ return { total_bytes: total, free_bytes: free, available_bytes: available, used_bytes: total - available };
5737
+ }
5738
+ async function sampleHostMemory(options = {}) {
5739
+ const total = totalmem();
5740
+ const free = freemem();
5741
+ const fallback = { total_bytes: total, free_bytes: free, used_bytes: Math.max(0, total - free) };
5742
+ if ((options.platform ?? process.platform) !== "darwin") return fallback;
5743
+ return new Promise((resolve12) => {
5744
+ execFile("/usr/bin/vm_stat", [], {
5745
+ env: options.env ?? process.env,
5746
+ timeout: 1e3
5747
+ }, (error, stdout) => resolve12(error ? fallback : parseMacMemory(String(stdout), total) ?? fallback));
5748
+ });
4689
5749
  }
4690
- var FOUNDATION_DEFAULT_DEPTH = 2;
4691
- var FOUNDATION_MAX_DEPTH = 64;
4692
- var FOUNDATION_DEFAULT_VOCAB = 256;
4693
- var FOUNDATION_DEFAULT_SEQUENCE = 64;
4694
- var FOUNDATION_DEFAULT_BATCH = 2;
4695
- var ADAPTER_MEMORY_PROFILES = [
4696
- {
4697
- id: "Qwen/Qwen3.5-2B",
4698
- diskBytes: gib(8),
4699
- inferenceBytes: gib(8),
4700
- trainBytes: gib(16),
4701
- sparkClass: false
4702
- },
4703
- {
4704
- id: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
4705
- diskBytes: gib(80),
4706
- inferenceBytes: gib(80),
4707
- trainBytes: gib(96),
4708
- sparkClass: true
4709
- },
4710
- {
4711
- id: "meta-models/Muse-Glimmer-30B",
4712
- diskBytes: gib(80),
4713
- inferenceBytes: gib(80),
4714
- trainBytes: gib(96),
4715
- sparkClass: true
5750
+ function cpuTimes() {
5751
+ let idle = 0;
5752
+ let total = 0;
5753
+ for (const cpu of cpus()) {
5754
+ const times = cpu.times;
5755
+ idle += times.idle;
5756
+ total += times.user + times.nice + times.sys + times.irq + times.idle;
4716
5757
  }
4717
- ];
4718
- function adapterMemoryProfile(modelId) {
4719
- const normalized = modelId.trim().toLowerCase();
4720
- return ADAPTER_MEMORY_PROFILES.find((profile) => profile.id.toLowerCase() === normalized);
4721
- }
4722
- function foundationWidth(depth) {
4723
- return 32 * depth;
4724
- }
4725
- function foundationParamCount(depth, vocabSize = FOUNDATION_DEFAULT_VOCAB, sequenceLength = FOUNDATION_DEFAULT_SEQUENCE) {
4726
- const width = foundationWidth(depth);
4727
- return 12 * depth * width * width + vocabSize * width + sequenceLength * width;
4728
- }
4729
- function foundationTrainBytes(depth, options = {}) {
4730
- const vocabSize = options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB;
4731
- const sequenceLength = options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE;
4732
- const batchSize = options.batchSize ?? FOUNDATION_DEFAULT_BATCH;
4733
- const params = foundationParamCount(depth, vocabSize, sequenceLength);
4734
- const width = foundationWidth(depth);
4735
- const activationSlack = batchSize * sequenceLength * width * depth * 64;
4736
- return 16 * params + activationSlack;
4737
- }
4738
- function foundationInferenceBytes(depth, options = {}) {
4739
- const params = foundationParamCount(
4740
- depth,
4741
- options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB,
4742
- options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE
4743
- );
4744
- return 4 * params + gib(0.25);
5758
+ return { idle, total };
4745
5759
  }
4746
- function isUnifiedGpuName(name) {
4747
- if (!name) return false;
4748
- return /spark|gb10/i.test(name);
5760
+ function cpuUtilization(before, after) {
5761
+ const total = after.total - before.total;
5762
+ if (!(total > 0)) return void 0;
5763
+ const busy = total - (after.idle - before.idle);
5764
+ return Math.min(100, Math.max(0, Math.round(busy / total * 100)));
4749
5765
  }
4750
-
4751
- // src/local-runtime/capability.ts
4752
- var HEADROOM = 1.2;
4753
- var FIT_RATIO = 0.7;
4754
- function formatGiB(bytes) {
4755
- const amount = bytes / 1024 ** 3;
4756
- const precision = amount >= 10 ? 0 : 1;
4757
- return `${amount.toFixed(precision)} GiB`;
5766
+ function optionalNumber(value) {
5767
+ if (value === void 0) return void 0;
5768
+ const trimmed = value.trim();
5769
+ if (!trimmed || /^\[?n\/a\]?$/i.test(trimmed) || /not supported/i.test(trimmed)) return void 0;
5770
+ const parsed = Number(trimmed);
5771
+ return Number.isFinite(parsed) ? parsed : void 0;
4758
5772
  }
4759
- function compareFit(available, required, missing) {
4760
- if (available === void 0) {
4761
- return { status: "not_possible", reason: missing, required_bytes: required };
4762
- }
4763
- if (available < required) {
4764
- return {
4765
- status: "not_possible",
4766
- reason: `needs ~${formatGiB(required)}; ${formatGiB(available)} available`,
4767
- required_bytes: required,
4768
- available_bytes: available
4769
- };
4770
- }
4771
- if (available < required * HEADROOM) {
4772
- return {
4773
- status: "tight",
4774
- reason: `fits in ${formatGiB(available)} with little headroom (needs ~${formatGiB(required)})`,
4775
- required_bytes: required,
4776
- available_bytes: available
4777
- };
5773
+ var MIB = 1024 * 1024;
5774
+ var NVIDIA_USAGE_QUERY = [
5775
+ "--query-gpu=index,name,utilization.gpu,memory.used,memory.total,temperature.gpu",
5776
+ "--format=csv,noheader,nounits"
5777
+ ];
5778
+ function parseNvidiaUsage(stdout) {
5779
+ const gpus = [];
5780
+ for (const line of stdout.split(/\r?\n/).map((item) => item.trim()).filter(Boolean)) {
5781
+ const parts = line.split(",").map((part) => part.trim());
5782
+ if (parts.length < 2 || !/^\d+$/.test(parts[0] ?? "")) continue;
5783
+ const name = parts[1] || "NVIDIA GPU";
5784
+ const used = optionalNumber(parts[3]);
5785
+ const total = optionalNumber(parts[4]);
5786
+ gpus.push({
5787
+ index: Number(parts[0]),
5788
+ name,
5789
+ utilization_percent: optionalNumber(parts[2]),
5790
+ memory_used_bytes: used === void 0 ? void 0 : Math.round(used * MIB),
5791
+ memory_total_bytes: total === void 0 || total <= 0 ? void 0 : Math.round(total * MIB),
5792
+ temperature_c: optionalNumber(parts[5]),
5793
+ unified_memory: /spark|gb10/i.test(name),
5794
+ vendor: "nvidia"
5795
+ });
4778
5796
  }
4779
- return {
4780
- status: "ready",
4781
- reason: `${formatGiB(available)} available (needs ~${formatGiB(required)})`,
4782
- required_bytes: required,
4783
- available_bytes: available
4784
- };
5797
+ return gpus;
4785
5798
  }
4786
- function trainingBlockers(inventory) {
4787
- const blockers = [];
4788
- if (!inventory.node.ok) blockers.push(`Node ${inventory.node.version} is below 22`);
4789
- if (inventory.uv && !inventory.uv.ok) blockers.push(inventory.uv.message);
4790
- if (inventory.python && !inventory.python.ok) blockers.push(inventory.python.message);
4791
- return blockers;
5799
+ function queryNvidia(env, timeoutMs) {
5800
+ return new Promise((resolve12) => {
5801
+ execFile("nvidia-smi", NVIDIA_USAGE_QUERY, { env, timeout: timeoutMs, windowsHide: true }, (error, stdout) => {
5802
+ if (error) {
5803
+ const code = error.code;
5804
+ resolve12({
5805
+ gpus: [],
5806
+ note: code === "ENOENT" ? "nvidia-smi not found" : error.killed ? "nvidia-smi timed out" : "nvidia-smi failed"
5807
+ });
5808
+ return;
5809
+ }
5810
+ const gpus = parseNvidiaUsage(String(stdout));
5811
+ resolve12(gpus.length > 0 ? { gpus } : { gpus, note: "nvidia-smi listed no GPUs" });
5812
+ });
5813
+ });
4792
5814
  }
4793
- function cudaAvailable(inventory) {
4794
- if (inventory.python?.ok && inventory.python.cuda_available !== void 0) {
4795
- return inventory.python.cuda_available;
5815
+ function tidyCpuModel(model) {
5816
+ return model.replace(/\((?:R|TM|tm|r)\)/g, "").replace(/\s+(?:CPU|Processor)\b/gi, "").replace(/\s*@\s*[\d.]+\s*[GM]Hz/i, "").replace(/\s+\d+-Core$/i, "").replace(/\s+/g, " ").trim() || "CPU";
5817
+ }
5818
+ var ARM_CORES = {
5819
+ "0xd03": "Cortex-A53",
5820
+ "0xd05": "Cortex-A55",
5821
+ "0xd08": "Cortex-A72",
5822
+ "0xd0b": "Cortex-A76",
5823
+ "0xd0c": "Neoverse-N1",
5824
+ "0xd40": "Neoverse-V1",
5825
+ "0xd41": "Cortex-A78",
5826
+ "0xd44": "Cortex-X1",
5827
+ "0xd47": "Cortex-A710",
5828
+ "0xd48": "Cortex-X2",
5829
+ "0xd49": "Neoverse-N2",
5830
+ "0xd4d": "Cortex-A715",
5831
+ "0xd4e": "Cortex-X3",
5832
+ "0xd4f": "Neoverse-V2",
5833
+ "0xd80": "Cortex-A520",
5834
+ "0xd81": "Cortex-A720",
5835
+ "0xd82": "Cortex-X4",
5836
+ "0xd84": "Neoverse-V3",
5837
+ "0xd85": "Cortex-X925",
5838
+ "0xd87": "Cortex-A725",
5839
+ "0xd8e": "Neoverse-N3"
5840
+ };
5841
+ function armCpuModel(cpuinfo) {
5842
+ const names = [];
5843
+ let implementer;
5844
+ for (const line of cpuinfo.split(/\r?\n/)) {
5845
+ const [key, value] = line.split(":").map((item) => item.trim().toLowerCase());
5846
+ if (key === "cpu implementer") implementer = value;
5847
+ if (key !== "cpu part" || !value) continue;
5848
+ const name = implementer === "0x41" ? ARM_CORES[value] : void 0;
5849
+ if (name && !names.includes(name)) names.push(name);
5850
+ }
5851
+ return names.length > 0 ? names.join(" + ") : void 0;
5852
+ }
5853
+ function cpuModel(reported, platform, arch) {
5854
+ const model = tidyCpuModel(reported ?? "");
5855
+ if (platform !== "linux" || !/^(?:unknown|CPU)$/i.test(model)) return model;
5856
+ try {
5857
+ return armCpuModel(readFileSync("/proc/cpuinfo", "utf8")) ?? (arch === "arm64" ? "Arm CPU" : model);
5858
+ } catch {
5859
+ return arch === "arm64" ? "Arm CPU" : model;
4796
5860
  }
4797
- return inventory.nvidia_smi.ok && inventory.gpus.length > 0;
4798
5861
  }
4799
- function trainingGpu(inventory) {
4800
- const listed = inventory.gpus.find((gpu) => gpu.index === 0) ?? inventory.gpus[0];
4801
- if (listed) {
4802
- if (!listed.memory_total_bytes && inventory.python?.total_memory_bytes) {
4803
- return { ...listed, memory_total_bytes: inventory.python.total_memory_bytes };
4804
- }
4805
- if (!listed.memory_total_bytes && (listed.unified_memory || isUnifiedGpuName(listed.name))) {
4806
- return { ...listed, memory_total_bytes: inventory.os.total_memory_bytes, unified_memory: true };
5862
+ async function modelCacheDisk(env) {
5863
+ const path = resolve10(env.HF_HOME?.trim() || join13(homedir3(), ".cache", "huggingface"));
5864
+ for (let probe = path, depth = 0; depth < 8; depth += 1) {
5865
+ try {
5866
+ const fs = await statfs(probe);
5867
+ return {
5868
+ path,
5869
+ free_bytes: Number(fs.bavail) * Number(fs.bsize),
5870
+ total_bytes: Number(fs.blocks) * Number(fs.bsize)
5871
+ };
5872
+ } catch {
5873
+ const parent = dirname9(probe);
5874
+ if (parent === probe) break;
5875
+ probe = parent;
4807
5876
  }
4808
- return listed;
4809
- }
4810
- if (inventory.python?.cuda_device) {
4811
- return {
4812
- index: 0,
4813
- name: inventory.python.cuda_device,
4814
- memory_total_bytes: inventory.python.total_memory_bytes,
4815
- unified_memory: isUnifiedGpuName(inventory.python.cuda_device)
4816
- };
4817
5877
  }
4818
5878
  return void 0;
4819
5879
  }
4820
- function gpuMemory(gpu) {
4821
- return gpu?.memory_total_bytes;
4822
- }
4823
- function diskForCache(inventory) {
4824
- return inventory.disks.find((disk) => disk.name === "model-cache")?.free_bytes;
4825
- }
4826
- function worse(left, right) {
4827
- const rank = { not_possible: 2, tight: 1, ready: 0 };
4828
- return rank[left.status] >= rank[right.status] ? left : right;
4829
- }
4830
- function withBlockers(verdict, blockers) {
4831
- if (blockers.length === 0) return verdict;
5880
+ function appleGpu(cpuModel2, platform, arch) {
5881
+ if (platform !== "darwin" || arch !== "arm64") return void 0;
5882
+ const chip = /apple/i.test(cpuModel2) ? cpuModel2.trim() : "Apple Silicon";
4832
5883
  return {
4833
- status: "not_possible",
4834
- reason: blockers.join("; "),
4835
- required_bytes: verdict.required_bytes,
4836
- available_bytes: verdict.available_bytes
5884
+ index: 0,
5885
+ name: `${chip} GPU`,
5886
+ memory_total_bytes: totalmem(),
5887
+ unified_memory: true,
5888
+ vendor: "apple"
4837
5889
  };
4838
5890
  }
4839
- function adapterVerdicts(profile, inventory, gpu, hasCuda) {
4840
- const blockers = trainingBlockers(inventory);
4841
- const gpuBytes = gpuMemory(gpu);
4842
- const ram = inventory.os.total_memory_bytes;
4843
- const disk = diskForCache(inventory);
4844
- let train;
4845
- if (!hasCuda) {
4846
- train = {
4847
- status: "not_possible",
4848
- reason: "LoRA training requires CUDA",
4849
- required_bytes: profile.trainBytes
4850
- };
4851
- } else {
4852
- train = compareFit(gpuBytes, profile.trainBytes, "GPU memory is unknown");
4853
- }
4854
- train = withBlockers(train, blockers);
4855
- if (disk !== void 0 && disk < profile.diskBytes && train.status !== "not_possible") {
4856
- train = worse(train, compareFit(disk, profile.diskBytes, "model cache disk is unknown"));
4857
- }
4858
- const finetune = train;
4859
- let inference;
4860
- if (hasCuda) {
4861
- inference = compareFit(gpuBytes, profile.inferenceBytes, "GPU memory is unknown");
4862
- } else {
4863
- inference = compareFit(ram, profile.inferenceBytes, "system RAM is unknown");
4864
- if (inference.status !== "not_possible") {
4865
- inference = {
4866
- ...inference,
4867
- reason: `CPU inference: ${inference.reason}`
4868
- };
4869
- }
4870
- }
5891
+ async function sampleLiveUsage(options = {}) {
5892
+ const env = options.env ?? process.env;
5893
+ const platform = options.platform ?? process.platform;
5894
+ const arch = options.arch ?? process.arch;
5895
+ const before = cpuTimes();
5896
+ const [nvidia, disk, memory] = await Promise.all([
5897
+ queryNvidia(env, options.gpuTimeoutMs ?? 2e3),
5898
+ modelCacheDisk(env).catch(() => void 0),
5899
+ sampleHostMemory({ env, platform }),
5900
+ new Promise((resolve12) => setTimeout(resolve12, options.cpuSampleMs ?? 200))
5901
+ ]);
5902
+ const after = cpuTimes();
5903
+ const list = cpus();
5904
+ const model = cpuModel(list[0]?.model, platform, arch);
5905
+ const apple = nvidia.gpus.length === 0 ? appleGpu(model, platform, arch) : void 0;
5906
+ const [one, five, fifteen] = loadavg();
4871
5907
  return {
4872
- id: profile.id,
4873
- spark_class: profile.sparkClass,
4874
- train,
4875
- finetune,
4876
- inference
5908
+ sampled_at: (/* @__PURE__ */ new Date()).toISOString(),
5909
+ platform,
5910
+ arch,
5911
+ cpu: {
5912
+ model,
5913
+ cores: list.length,
5914
+ utilization_percent: cpuUtilization(before, after),
5915
+ load_average: [one ?? 0, five ?? 0, fifteen ?? 0]
5916
+ },
5917
+ memory,
5918
+ disk,
5919
+ gpus: apple ? [apple] : nvidia.gpus,
5920
+ gpu_note: apple ? void 0 : nvidia.note
4877
5921
  };
4878
5922
  }
4879
- function suggestedFoundationDepth(availableBytes) {
4880
- if (availableBytes === void 0) return FOUNDATION_DEFAULT_DEPTH;
4881
- const budget = availableBytes * FIT_RATIO;
4882
- let suggested = FOUNDATION_DEFAULT_DEPTH;
4883
- for (let depth = FOUNDATION_DEFAULT_DEPTH; depth <= FOUNDATION_MAX_DEPTH; depth += 1) {
4884
- if (foundationTrainBytes(depth) <= budget) suggested = depth;
4885
- else break;
4886
- }
4887
- return suggested;
4888
- }
4889
- function evaluateCapabilities(inventory) {
4890
- const hasCuda = cudaAvailable(inventory);
4891
- const gpu = trainingGpu(inventory);
4892
- const notes = [];
4893
- if (gpu?.unified_memory || isUnifiedGpuName(gpu?.name)) {
4894
- notes.push("GPU memory is a unified pool shared with system RAM (Spark-class).");
4895
- }
4896
- if (inventory.quick) {
4897
- notes.push("Quick probe skipped the bundled torch/uv runtime.");
4898
- }
4899
- if (!inventory.node.ok) {
4900
- notes.push(`Node ${inventory.node.version} is below the 22.x required by the local runner.`);
4901
- }
4902
- const adapters = ADAPTER_MEMORY_PROFILES.map(
4903
- (profile) => adapterVerdicts(profile, inventory, gpu, hasCuda)
4904
- );
4905
- const blockers = trainingBlockers(inventory);
4906
- const gpuBytes = gpuMemory(gpu);
4907
- const ram = inventory.os.total_memory_bytes;
4908
- const trainBudget = hasCuda ? gpuBytes : void 0;
4909
- const suggested = suggestedFoundationDepth(trainBudget);
4910
- const defaultTrain = foundationTrainBytes(FOUNDATION_DEFAULT_DEPTH);
4911
- const inferBudget = hasCuda ? gpuBytes : ram;
4912
- let train;
4913
- if (!hasCuda) {
4914
- train = {
4915
- status: "not_possible",
4916
- reason: "Foundation pretrain, SFT, and RL require CUDA",
4917
- required_bytes: defaultTrain
4918
- };
4919
- } else {
4920
- train = compareFit(gpuBytes, defaultTrain, "GPU memory is unknown");
4921
- }
4922
- train = withBlockers(train, blockers);
4923
- const inference = compareFit(
4924
- inferBudget,
4925
- foundationInferenceBytes(FOUNDATION_DEFAULT_DEPTH),
4926
- hasCuda ? "GPU memory is unknown" : "system RAM is unknown"
4927
- );
4928
- const foundation = {
4929
- default_depth: FOUNDATION_DEFAULT_DEPTH,
4930
- suggested_max_depth: hasCuda && blockers.length === 0 ? suggested : FOUNDATION_DEFAULT_DEPTH,
4931
- train,
4932
- finetune: train,
4933
- inference: hasCuda ? inference : { ...inference, reason: `CPU eval: ${inference.reason}` },
4934
- serve: hasCuda && inventory.os.platform === "linux" ? inference : { status: "not_possible", reason: "Foundation serving requires Linux and an NVIDIA CUDA GPU" }
4935
- };
4936
- return { cuda_available: hasCuda, gpu, adapters, foundation, notes };
4937
- }
4938
- function formatCapabilitySummary(report) {
4939
- const gpu = report.gpu;
4940
- const gpuLabel = gpu ? `${gpu.name}${gpu.memory_total_bytes ? `, ${formatGiB(gpu.memory_total_bytes)}` : ""}${gpu.unified_memory ? " unified" : ""}` : "no NVIDIA GPU";
4941
- const adapterBits = report.adapters.map((adapter) => {
4942
- const short = adapter.id.split("/").at(-1) ?? adapter.id;
4943
- return `${short} train ${adapter.train.status}`;
4944
- });
4945
- return [
4946
- `GPU ${gpuLabel}, CUDA ${report.cuda_available ? "yes" : "no"}`,
4947
- `Adapter: ${adapterBits.join("; ")}`,
4948
- `Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve ${report.foundation.serve.status}`
4949
- ].join(". ");
4950
- }
4951
- function warningsFromSnapshot(report, target) {
4952
- const warnings = [];
4953
- if (target.engine === "decision") {
4954
- if (!report.cuda_available) {
4955
- warnings.push("No CUDA GPU detected: decision training runs on CPU or Apple MPS (roughly 15 s per 8-example step on a 4-core CPU).");
4956
- }
4957
- return warnings;
4958
- }
4959
- if (target.engine === "foundation") {
4960
- for (const kind of ["train", "finetune"]) {
4961
- const verdict = report.foundation[kind];
4962
- if (verdict.status === "not_possible" || verdict.status === "tight") {
4963
- warnings.push(`Foundation ${kind} is ${verdict.status}: ${verdict.reason}`);
4964
- }
4965
- }
4966
- return warnings;
4967
- }
4968
- const modelId = target.baseModel;
4969
- const adapter = modelId ? report.adapters.find((item) => item.id.toLowerCase() === modelId.trim().toLowerCase()) : void 0;
4970
- if (!adapter) {
4971
- if (modelId && !adapterMemoryProfile(modelId)) return warnings;
4972
- const blocked = report.adapters.filter((item) => item.train.status === "not_possible");
4973
- if (blocked.length === report.adapters.length) {
4974
- warnings.push(`Adapter train is not_possible: ${report.adapters[0]?.train.reason ?? "CUDA required"}`);
4975
- }
4976
- return warnings;
4977
- }
4978
- for (const kind of ["train", "finetune", "inference"]) {
4979
- const verdict = adapter[kind];
4980
- if (verdict.status === "not_possible" || verdict.status === "tight") {
4981
- warnings.push(`${adapter.id} ${kind} is ${verdict.status}: ${verdict.reason}`);
4982
- }
4983
- }
4984
- return warnings;
4985
- }
4986
5923
 
4987
5924
  // src/local-runtime/host-inventory.ts
4988
- import { constants } from "fs";
4989
- import { access as access2, statfs } from "fs/promises";
4990
- import { cpus, freemem, homedir as homedir3, totalmem, type } from "os";
4991
- import { dirname as dirname7, join as join9, resolve as resolve9 } from "path";
4992
- import { spawn } from "child_process";
4993
5925
  async function runCommand(command, args, options = {}) {
4994
5926
  const timeoutMs = options.timeoutMs ?? 1e4;
4995
5927
  let interrupted;
@@ -5069,6 +6001,9 @@ async function runCommand(command, args, options = {}) {
5069
6001
  function firstLine(value) {
5070
6002
  return value.trim().split(/\r?\n/)[0] ?? "";
5071
6003
  }
6004
+ function supportsPackagedTrainingRuntime(os) {
6005
+ return os.platform === "linux" && ["x64", "arm64"].includes(os.arch) || os.platform === "darwin" && os.arch === "arm64" || os.platform === "win32" && os.arch === "x64";
6006
+ }
5072
6007
  function parseMiB(value) {
5073
6008
  if (!value) return void 0;
5074
6009
  const parsed = Number(value.replace(/[^\d.]/g, ""));
@@ -5142,10 +6077,10 @@ async function queryNvidiaGpus(env) {
5142
6077
  return { result: table.code === 0 || !last ? table : last, gpus };
5143
6078
  }
5144
6079
  async function diskCheck(name, path) {
5145
- const resolvedPath = resolve9(path);
6080
+ const resolvedPath = resolve11(path);
5146
6081
  try {
5147
- await access2(resolvedPath, constants.R_OK);
5148
- const fs = await statfs(resolvedPath);
6082
+ await access2(resolvedPath, constants5.R_OK);
6083
+ const fs = await statfs2(resolvedPath);
5149
6084
  const freeBytes = Number(fs.bavail) * Number(fs.bsize);
5150
6085
  return {
5151
6086
  name,
@@ -5157,11 +6092,11 @@ async function diskCheck(name, path) {
5157
6092
  } catch (error) {
5158
6093
  let probe = resolvedPath;
5159
6094
  for (let i = 0; i < 6; i += 1) {
5160
- const parent = dirname7(probe);
6095
+ const parent = dirname10(probe);
5161
6096
  if (parent === probe) break;
5162
6097
  probe = parent;
5163
6098
  try {
5164
- const fs = await statfs(probe);
6099
+ const fs = await statfs2(probe);
5165
6100
  const freeBytes = Number(fs.bavail) * Number(fs.bsize);
5166
6101
  return {
5167
6102
  name,
@@ -5182,11 +6117,13 @@ async function diskCheck(name, path) {
5182
6117
  };
5183
6118
  }
5184
6119
  }
5185
- function pythonProbeSource() {
6120
+ function pythonProbeSource(decision = false) {
5186
6121
  return [
5187
6122
  "import json",
5188
- "import torch, transformers, peft, huggingface_hub",
6123
+ decision ? "import torch, transformers, laya; peft = None" : "import torch, transformers, peft, huggingface_hub",
6124
+ "x = torch.ones(1, requires_grad=True); (x * x).sum().backward()",
5189
6125
  "payload = {'python_ok': True, 'torch': torch.__version__, 'transformers': transformers.__version__, 'peft': getattr(peft, '__version__', None), 'cuda_available': bool(torch.cuda.is_available())}",
6126
+ "payload['mps_available'] = bool(torch.backends.mps.is_available())",
5190
6127
  "cuda = torch.cuda.is_available()",
5191
6128
  "payload.update({'cuda_device': torch.cuda.get_device_name(0), 'compute_capability': list(torch.cuda.get_device_capability(0)), 'total_memory_bytes': torch.cuda.get_device_properties(0).total_memory, 'bf16_supported': bool(torch.cuda.is_bf16_supported())} if cuda else {})",
5192
6129
  "print(json.dumps(payload))"
@@ -5205,6 +6142,7 @@ function parsePythonProbeJson(text) {
5205
6142
  transformers: typeof parsed.transformers === "string" ? parsed.transformers : void 0,
5206
6143
  peft: typeof parsed.peft === "string" ? parsed.peft : void 0,
5207
6144
  cuda_available: parsed.cuda_available === true,
6145
+ mps_available: typeof parsed.mps_available === "boolean" ? parsed.mps_available : void 0,
5208
6146
  cuda_device: typeof parsed.cuda_device === "string" ? parsed.cuda_device : void 0,
5209
6147
  compute_capability: Array.isArray(parsed.compute_capability) ? parsed.compute_capability.filter((value) => typeof value === "number") : void 0,
5210
6148
  total_memory_bytes: typeof parsed.total_memory_bytes === "number" ? parsed.total_memory_bytes : void 0,
@@ -5214,9 +6152,9 @@ function parsePythonProbeJson(text) {
5214
6152
  return void 0;
5215
6153
  }
5216
6154
  }
5217
- async function collectPythonRuntime(config, env) {
5218
- const entrypoint = buildBundledPythonCommand("-c", [pythonProbeSource()]);
5219
- const childEnv = withBundledPythonEnvironment(
6155
+ async function collectPythonRuntime(config, env, decision = false) {
6156
+ const entrypoint = (decision ? buildDecisionPythonCommand : buildBundledPythonCommand)("-c", [pythonProbeSource(decision)]);
6157
+ const childEnv = (decision ? withDecisionPythonEnvironment : withBundledPythonEnvironment)(
5220
6158
  withHuggingFaceCacheEnvironment(
5221
6159
  minimalMachineLearningEnvironment(env),
5222
6160
  config?.paths.modelCache
@@ -5243,17 +6181,18 @@ async function collectHostInventory(options = {}) {
5243
6181
  const cwd = options.cwd ?? process.cwd();
5244
6182
  const nodeVersion = process.versions.node;
5245
6183
  const nodeMajor = Number(nodeVersion.split(".")[0]);
5246
- const artifactRoot = resolve9(cwd, config?.artifactRoot ?? join9(cwd, DEFAULT_ARTIFACT_ROOT));
5247
- const storeRoot = resolve9(config?.storeRoot ?? defaultStoreRoot(env));
5248
- const modelCache = resolve9(
5249
- config?.paths.modelCache ?? env.HF_HOME ?? join9(homedir3(), ".cache", "huggingface")
6184
+ const artifactRoot = resolve11(cwd, config?.artifactRoot ?? join14(cwd, DEFAULT_ARTIFACT_ROOT));
6185
+ const storeRoot = resolve11(config?.storeRoot ?? defaultStoreRoot(env));
6186
+ const modelCache = resolve11(
6187
+ config?.paths.modelCache ?? env.HF_HOME ?? join14(homedir4(), ".cache", "huggingface")
5250
6188
  );
5251
- const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried] = await Promise.all([
6189
+ const [artifactDisk, storeDisk, cacheDisk, uvVersion, nvidiaQueried, memory] = await Promise.all([
5252
6190
  diskCheck("artifact-root", artifactRoot),
5253
6191
  diskCheck("store-root", storeRoot),
5254
6192
  diskCheck("model-cache", modelCache),
5255
6193
  runCommand("uv", ["--version"], { env, timeoutMs: 1e4 }),
5256
- queryNvidiaGpus(env)
6194
+ queryNvidiaGpus(env),
6195
+ sampleHostMemory({ env })
5257
6196
  ]);
5258
6197
  const nvidia = nvidiaQueried.result;
5259
6198
  const gpus = nvidiaQueried.gpus;
@@ -5269,9 +6208,10 @@ async function collectHostInventory(options = {}) {
5269
6208
  platform: process.platform,
5270
6209
  arch: process.arch,
5271
6210
  type: type(),
5272
- cpu_count: cpus().length,
5273
- total_memory_bytes: totalmem(),
5274
- free_memory_bytes: freemem()
6211
+ cpu_count: cpus2().length,
6212
+ total_memory_bytes: memory.total_bytes,
6213
+ free_memory_bytes: memory.free_bytes ?? memory.total_bytes - memory.used_bytes,
6214
+ available_memory_bytes: memory.available_bytes
5275
6215
  },
5276
6216
  uv: {
5277
6217
  ok: uvVersion.code === 0,
@@ -5285,19 +6225,458 @@ async function collectHostInventory(options = {}) {
5285
6225
  },
5286
6226
  disks: [artifactDisk, storeDisk, cacheDisk]
5287
6227
  };
5288
- if (options.quick !== true) {
6228
+ if (options.quick !== true && !supportsPackagedTrainingRuntime(inventory.os)) {
6229
+ const unavailable = { ok: false, message: "No packaged training runtime for this OS/architecture." };
6230
+ inventory.python = unavailable;
6231
+ inventory.decision_python = unavailable;
6232
+ } else if (options.quick !== true) {
6233
+ inventory.decision_python = uvVersion.code === 0 ? await collectPythonRuntime(config, env, true) : { ok: false, message: "Decision runtime probe skipped because uv is not available" };
5289
6234
  inventory.python = uvVersion.code === 0 ? await collectPythonRuntime(config, env) : { ok: false, message: "Python probe skipped because uv is not available" };
5290
6235
  }
5291
6236
  return inventory;
5292
6237
  }
5293
6238
 
6239
+ // src/local-runtime/workflow-guidance.ts
6240
+ var WORKFLOW_STATUS_LABEL = {
6241
+ ready: "Ready",
6242
+ needs_setup: "Needs setup",
6243
+ limited: "Limited resources",
6244
+ remote: "Remote GPU",
6245
+ unverified: "Needs verification",
6246
+ unsupported: "Unsupported runtime"
6247
+ };
6248
+ var GiB = 1024 ** 3;
6249
+ function workflowGuidance(host, report) {
6250
+ const platform = host.os.platform;
6251
+ const supported = supportsPackagedTrainingRuntime(host.os);
6252
+ const setup = !host.node.ok ? "Install Node 22 or newer." : host.uv?.ok === false ? "Install uv to set up the training runtime." : void 0;
6253
+ const probe = host.decision_python;
6254
+ const device = probe?.cuda_available ? "CUDA" : probe?.mps_available ? "Apple MPS" : "CPU";
6255
+ const decision = {
6256
+ id: "decision",
6257
+ label: "Decision training",
6258
+ status: "unverified",
6259
+ reason: platform === "darwin" && host.os.arch === "arm64" ? "CPU supported; Apple MPS availability needs a runtime check." : "CPU supported; no NVIDIA GPU required.",
6260
+ next: "Run tt hardware, then try tt init --engine decision."
6261
+ };
6262
+ if (!supported) {
6263
+ Object.assign(decision, {
6264
+ status: "unsupported",
6265
+ reason: "No packaged decision runtime for this OS/architecture.",
6266
+ next: "Use a supported Linux or Apple Silicon machine; specs and data preparation remain available."
6267
+ });
6268
+ } else if (setup || probe?.ok === false) {
6269
+ Object.assign(decision, {
6270
+ status: "needs_setup",
6271
+ reason: setup ?? `Decision runtime check failed: ${probe.message}`,
6272
+ next: "Resolve the setup issue and rerun tt hardware."
6273
+ });
6274
+ } else if (probe?.ok) {
6275
+ Object.assign(decision, {
6276
+ status: platform === "win32" ? "unverified" : "ready",
6277
+ reason: `${device} runtime available; decision model fit is estimated.`,
6278
+ next: "Try tt init --engine decision; use a small dataset first."
6279
+ });
6280
+ }
6281
+ if (supported && decision.status !== "needs_setup") {
6282
+ const memory = device === "CUDA" && report.gpu && !report.gpu.unified_memory ? report.gpu.memory_free_bytes ?? report.gpu.memory_total_bytes : host.os.available_memory_bytes ?? host.os.free_memory_bytes;
6283
+ const lowDisk = host.disks.find((d) => d.name === "model-cache" && d.free_bytes !== void 0 && d.free_bytes < 6 * GiB);
6284
+ if (memory !== void 0 && memory < 8 * GiB || lowDisk || host.os.cpu_count < 4) {
6285
+ decision.status = "limited";
6286
+ decision.reason += lowDisk ? " Less than 6 GiB cache space free." : memory !== void 0 && memory < 8 * GiB ? host.os.available_memory_bytes !== void 0 && device !== "CUDA" ? " Less than 8 GiB estimated available RAM." : " Less than 8 GiB working memory free." : " Fewer than 4 CPU cores; training may be slow.";
6287
+ decision.next = "Start with a small batch; free memory/disk or try CPU with freeze_encoder for a smaller training workload.";
6288
+ }
6289
+ if (platform === "win32") decision.reason += " Native Windows training is not yet end-to-end certified.";
6290
+ }
6291
+ const remote = "Configure runtime.gpu for your AWS GPU; tt doctor checks it.";
6292
+ const training = (id, label, verdict) => {
6293
+ if (!supported) return {
6294
+ id,
6295
+ label,
6296
+ status: "remote",
6297
+ reason: "This packaged local training runtime is unavailable.",
6298
+ next: remote
6299
+ };
6300
+ if (setup || host.python?.ok === false) return {
6301
+ id,
6302
+ label,
6303
+ status: "needs_setup",
6304
+ reason: setup ?? host.python.message,
6305
+ next: "Fix the local runtime, or configure runtime.gpu for AWS."
6306
+ };
6307
+ if (!report.cuda_available) return {
6308
+ id,
6309
+ label,
6310
+ status: "remote",
6311
+ reason: "Current TT training requires NVIDIA CUDA.",
6312
+ next: remote
6313
+ };
6314
+ if (platform !== "linux") return {
6315
+ id,
6316
+ label,
6317
+ status: "unverified",
6318
+ reason: "Native training on this OS is not certified; Windows users can use a Linux/WSL2 environment.",
6319
+ next: "Run tt doctor in the training environment, or use your AWS GPU."
6320
+ };
6321
+ if (verdict.status !== "ready") return {
6322
+ id,
6323
+ label,
6324
+ status: "limited",
6325
+ reason: verdict.reason,
6326
+ next: "Choose a smaller supported model/configuration or use your AWS GPU."
6327
+ };
6328
+ const free = report.gpu?.unified_memory ? host.os.free_memory_bytes : report.gpu?.memory_free_bytes;
6329
+ if (free !== void 0 && verdict.required_bytes !== void 0 && free < verdict.required_bytes) {
6330
+ return {
6331
+ id,
6332
+ label,
6333
+ status: "limited",
6334
+ reason: `Available GPU memory is below the estimated workload requirement (${(free / GiB).toFixed(1)} GiB free).`,
6335
+ next: "Free GPU/shared memory, choose a smaller configuration, or use your AWS GPU."
6336
+ };
6337
+ }
6338
+ const verified = !host.quick && host.python?.ok;
6339
+ return {
6340
+ id,
6341
+ label,
6342
+ status: verified ? "ready" : "unverified",
6343
+ reason: verdict.reason,
6344
+ next: verified ? "tt doctor --spec tunedtensor.json checks the selected configuration." : "Run tt hardware to verify the training runtime."
6345
+ };
6346
+ };
6347
+ const best = report.adapters.find((a) => a.finetune.status === "ready") ?? report.adapters.find((a) => a.finetune.status === "tight") ?? report.adapters[0];
6348
+ return [
6349
+ {
6350
+ id: "prepare",
6351
+ label: "Specs and datasets",
6352
+ status: host.node.ok ? "ready" : "needs_setup",
6353
+ reason: "Create specs, inspect and split local data; no GPU required.",
6354
+ next: host.node.ok ? "Ask TT to prepare a dataset or create a spec. Agent chat requires managed access or your own model provider." : "Install Node 22 or newer."
6355
+ },
6356
+ decision,
6357
+ training("adapter", "LLM fine-tuning", { ...best.finetune, reason: `${best.id}: ${best.finetune.reason}` }),
6358
+ training("foundation", "Train from scratch", report.foundation.train),
6359
+ {
6360
+ id: "serve",
6361
+ label: "LLM serving",
6362
+ status: platform === "linux" && report.cuda_available ? "unverified" : "remote",
6363
+ reason: "tt serve uses a separate Linux/NVIDIA CUDA runtime; CPU evaluation does not imply CPU serving.",
6364
+ next: "On the GPU machine, use tt serve <model-id> --print-command to check the launch plan; actual startup checks the serving runtime."
6365
+ }
6366
+ ];
6367
+ }
6368
+
6369
+ // src/local-runtime/capability-profiles.ts
6370
+ var GIB = 1024 ** 3;
6371
+ function gib(n) {
6372
+ return Math.round(n * GIB);
6373
+ }
6374
+ var FOUNDATION_DEFAULT_DEPTH = 2;
6375
+ var FOUNDATION_MAX_DEPTH = 64;
6376
+ var FOUNDATION_DEFAULT_VOCAB = 256;
6377
+ var FOUNDATION_DEFAULT_SEQUENCE = 64;
6378
+ var FOUNDATION_DEFAULT_BATCH = 2;
6379
+ var ADAPTER_MEMORY_PROFILES = [
6380
+ {
6381
+ id: "Qwen/Qwen3.5-2B",
6382
+ diskBytes: gib(8),
6383
+ inferenceBytes: gib(8),
6384
+ trainBytes: gib(16),
6385
+ sparkClass: false
6386
+ },
6387
+ {
6388
+ id: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
6389
+ diskBytes: gib(80),
6390
+ inferenceBytes: gib(80),
6391
+ trainBytes: gib(96),
6392
+ sparkClass: true
6393
+ },
6394
+ {
6395
+ id: "meta-models/Muse-Glimmer-30B",
6396
+ diskBytes: gib(80),
6397
+ inferenceBytes: gib(80),
6398
+ trainBytes: gib(96),
6399
+ sparkClass: true
6400
+ }
6401
+ ];
6402
+ function adapterMemoryProfile(modelId) {
6403
+ const normalized = modelId.trim().toLowerCase();
6404
+ return ADAPTER_MEMORY_PROFILES.find((profile) => profile.id.toLowerCase() === normalized);
6405
+ }
6406
+ function foundationWidth(depth) {
6407
+ return 32 * depth;
6408
+ }
6409
+ function foundationParamCount(depth, vocabSize = FOUNDATION_DEFAULT_VOCAB, sequenceLength = FOUNDATION_DEFAULT_SEQUENCE) {
6410
+ const width = foundationWidth(depth);
6411
+ return 12 * depth * width * width + vocabSize * width + sequenceLength * width;
6412
+ }
6413
+ function foundationTrainBytes(depth, options = {}) {
6414
+ const vocabSize = options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB;
6415
+ const sequenceLength = options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE;
6416
+ const batchSize = options.batchSize ?? FOUNDATION_DEFAULT_BATCH;
6417
+ const params = foundationParamCount(depth, vocabSize, sequenceLength);
6418
+ const width = foundationWidth(depth);
6419
+ const activationSlack = batchSize * sequenceLength * width * depth * 64;
6420
+ return 16 * params + activationSlack;
6421
+ }
6422
+ function foundationInferenceBytes(depth, options = {}) {
6423
+ const params = foundationParamCount(
6424
+ depth,
6425
+ options.vocabSize ?? FOUNDATION_DEFAULT_VOCAB,
6426
+ options.sequenceLength ?? FOUNDATION_DEFAULT_SEQUENCE
6427
+ );
6428
+ return 4 * params + gib(0.25);
6429
+ }
6430
+ function isUnifiedGpuName(name) {
6431
+ if (!name) return false;
6432
+ return /spark|gb10/i.test(name);
6433
+ }
6434
+
6435
+ // src/local-runtime/capability.ts
6436
+ var HEADROOM = 1.2;
6437
+ var FIT_RATIO = 0.7;
6438
+ function formatGiB(bytes) {
6439
+ const amount = bytes / 1024 ** 3;
6440
+ const precision = amount >= 10 ? 0 : 1;
6441
+ return `${amount.toFixed(precision)} GiB`;
6442
+ }
6443
+ function compareFit(available, required, missing) {
6444
+ if (available === void 0) {
6445
+ return { status: "not_possible", reason: missing, required_bytes: required };
6446
+ }
6447
+ if (available < required) {
6448
+ return {
6449
+ status: "not_possible",
6450
+ reason: `needs ~${formatGiB(required)}; ${formatGiB(available)} available`,
6451
+ required_bytes: required,
6452
+ available_bytes: available
6453
+ };
6454
+ }
6455
+ if (available < required * HEADROOM) {
6456
+ return {
6457
+ status: "tight",
6458
+ reason: `fits in ${formatGiB(available)} with little headroom (needs ~${formatGiB(required)})`,
6459
+ required_bytes: required,
6460
+ available_bytes: available
6461
+ };
6462
+ }
6463
+ return {
6464
+ status: "ready",
6465
+ reason: `${formatGiB(available)} available (needs ~${formatGiB(required)})`,
6466
+ required_bytes: required,
6467
+ available_bytes: available
6468
+ };
6469
+ }
6470
+ function trainingBlockers(inventory) {
6471
+ const blockers = [];
6472
+ if (!inventory.node.ok) blockers.push(`Node ${inventory.node.version} is below 22`);
6473
+ if (inventory.uv && !inventory.uv.ok) blockers.push(inventory.uv.message);
6474
+ if (inventory.python && !inventory.python.ok) blockers.push(inventory.python.message);
6475
+ return blockers;
6476
+ }
6477
+ function cudaAvailable(inventory) {
6478
+ if (inventory.python?.ok && inventory.python.cuda_available !== void 0) {
6479
+ return inventory.python.cuda_available;
6480
+ }
6481
+ return inventory.nvidia_smi.ok && inventory.gpus.length > 0;
6482
+ }
6483
+ function trainingGpu(inventory) {
6484
+ const listed = inventory.gpus.find((gpu) => gpu.index === 0) ?? inventory.gpus[0];
6485
+ if (listed) {
6486
+ if (!listed.memory_total_bytes && inventory.python?.total_memory_bytes) {
6487
+ return { ...listed, memory_total_bytes: inventory.python.total_memory_bytes };
6488
+ }
6489
+ if (!listed.memory_total_bytes && (listed.unified_memory || isUnifiedGpuName(listed.name))) {
6490
+ return { ...listed, memory_total_bytes: inventory.os.total_memory_bytes, unified_memory: true };
6491
+ }
6492
+ return listed;
6493
+ }
6494
+ if (inventory.python?.cuda_device) {
6495
+ return {
6496
+ index: 0,
6497
+ name: inventory.python.cuda_device,
6498
+ memory_total_bytes: inventory.python.total_memory_bytes,
6499
+ unified_memory: isUnifiedGpuName(inventory.python.cuda_device)
6500
+ };
6501
+ }
6502
+ return void 0;
6503
+ }
6504
+ function gpuMemory(gpu) {
6505
+ return gpu?.memory_total_bytes;
6506
+ }
6507
+ function diskForCache(inventory) {
6508
+ return inventory.disks.find((disk) => disk.name === "model-cache")?.free_bytes;
6509
+ }
6510
+ function worse(left, right) {
6511
+ const rank = { not_possible: 2, tight: 1, ready: 0 };
6512
+ return rank[left.status] >= rank[right.status] ? left : right;
6513
+ }
6514
+ function withBlockers(verdict, blockers) {
6515
+ if (blockers.length === 0) return verdict;
6516
+ return {
6517
+ status: "not_possible",
6518
+ reason: blockers.join("; "),
6519
+ required_bytes: verdict.required_bytes,
6520
+ available_bytes: verdict.available_bytes
6521
+ };
6522
+ }
6523
+ function adapterVerdicts(profile, inventory, gpu, hasCuda) {
6524
+ const blockers = trainingBlockers(inventory);
6525
+ const gpuBytes = gpuMemory(gpu);
6526
+ const ram = inventory.os.total_memory_bytes;
6527
+ const disk = diskForCache(inventory);
6528
+ let train;
6529
+ if (!hasCuda) {
6530
+ train = {
6531
+ status: "not_possible",
6532
+ reason: "LoRA training requires CUDA",
6533
+ required_bytes: profile.trainBytes
6534
+ };
6535
+ } else {
6536
+ train = compareFit(gpuBytes, profile.trainBytes, "GPU memory is unknown");
6537
+ }
6538
+ train = withBlockers(train, blockers);
6539
+ if (disk !== void 0 && disk < profile.diskBytes && train.status !== "not_possible") {
6540
+ train = worse(train, compareFit(disk, profile.diskBytes, "model cache disk is unknown"));
6541
+ }
6542
+ const finetune = train;
6543
+ let inference;
6544
+ if (hasCuda) {
6545
+ inference = compareFit(gpuBytes, profile.inferenceBytes, "GPU memory is unknown");
6546
+ } else {
6547
+ inference = compareFit(ram, profile.inferenceBytes, "system RAM is unknown");
6548
+ if (inference.status !== "not_possible") {
6549
+ inference = {
6550
+ ...inference,
6551
+ reason: `CPU inference: ${inference.reason}`
6552
+ };
6553
+ }
6554
+ }
6555
+ return {
6556
+ id: profile.id,
6557
+ spark_class: profile.sparkClass,
6558
+ train,
6559
+ finetune,
6560
+ inference
6561
+ };
6562
+ }
6563
+ function suggestedFoundationDepth(availableBytes) {
6564
+ if (availableBytes === void 0) return FOUNDATION_DEFAULT_DEPTH;
6565
+ const budget = availableBytes * FIT_RATIO;
6566
+ let suggested = FOUNDATION_DEFAULT_DEPTH;
6567
+ for (let depth = FOUNDATION_DEFAULT_DEPTH; depth <= FOUNDATION_MAX_DEPTH; depth += 1) {
6568
+ if (foundationTrainBytes(depth) <= budget) suggested = depth;
6569
+ else break;
6570
+ }
6571
+ return suggested;
6572
+ }
6573
+ function evaluateCapabilities(inventory) {
6574
+ const hasCuda = cudaAvailable(inventory);
6575
+ const gpu = trainingGpu(inventory);
6576
+ const notes = [];
6577
+ if (gpu?.unified_memory || isUnifiedGpuName(gpu?.name)) {
6578
+ notes.push("GPU memory is a unified pool shared with system RAM (Spark-class).");
6579
+ }
6580
+ if (inventory.quick) {
6581
+ notes.push("Quick probe skipped the bundled torch/uv runtime.");
6582
+ }
6583
+ if (!inventory.node.ok) {
6584
+ notes.push(`Node ${inventory.node.version} is below the 22.x required by the local runner.`);
6585
+ }
6586
+ const adapters = ADAPTER_MEMORY_PROFILES.map(
6587
+ (profile) => adapterVerdicts(profile, inventory, gpu, hasCuda)
6588
+ );
6589
+ const blockers = trainingBlockers(inventory);
6590
+ const gpuBytes = gpuMemory(gpu);
6591
+ const ram = inventory.os.total_memory_bytes;
6592
+ const trainBudget = hasCuda ? gpuBytes : void 0;
6593
+ const suggested = suggestedFoundationDepth(trainBudget);
6594
+ const defaultTrain = foundationTrainBytes(FOUNDATION_DEFAULT_DEPTH);
6595
+ const inferBudget = hasCuda ? gpuBytes : ram;
6596
+ let train;
6597
+ if (!hasCuda) {
6598
+ train = {
6599
+ status: "not_possible",
6600
+ reason: "Foundation pretrain, SFT, and RL require CUDA",
6601
+ required_bytes: defaultTrain
6602
+ };
6603
+ } else {
6604
+ train = compareFit(gpuBytes, defaultTrain, "GPU memory is unknown");
6605
+ }
6606
+ train = withBlockers(train, blockers);
6607
+ const inference = compareFit(
6608
+ inferBudget,
6609
+ foundationInferenceBytes(FOUNDATION_DEFAULT_DEPTH),
6610
+ hasCuda ? "GPU memory is unknown" : "system RAM is unknown"
6611
+ );
6612
+ const foundation = {
6613
+ default_depth: FOUNDATION_DEFAULT_DEPTH,
6614
+ suggested_max_depth: hasCuda && blockers.length === 0 ? suggested : FOUNDATION_DEFAULT_DEPTH,
6615
+ train,
6616
+ finetune: train,
6617
+ inference: hasCuda ? inference : { ...inference, reason: `CPU eval: ${inference.reason}` },
6618
+ serve: hasCuda && inventory.os.platform === "linux" ? inference : { status: "not_possible", reason: "Foundation serving requires Linux and an NVIDIA CUDA GPU" }
6619
+ };
6620
+ const report = { cuda_available: hasCuda, gpu, adapters, foundation, notes };
6621
+ report.workflows = workflowGuidance(inventory, report);
6622
+ return report;
6623
+ }
6624
+ function formatCapabilitySummary(report) {
6625
+ const gpu = report.gpu;
6626
+ const gpuLabel = gpu ? `${gpu.name}${gpu.memory_total_bytes ? `, ${formatGiB(gpu.memory_total_bytes)}` : ""}${gpu.unified_memory ? " unified" : ""}` : "no NVIDIA GPU";
6627
+ const adapterBits = report.adapters.map((adapter) => {
6628
+ const short = adapter.id.split("/").at(-1) ?? adapter.id;
6629
+ return `${short} train ${adapter.train.status}`;
6630
+ });
6631
+ return [
6632
+ ...(report.workflows ?? []).map((item) => `${item.label}: ${WORKFLOW_STATUS_LABEL[item.status]} \u2014 ${item.reason} ${item.next}`),
6633
+ `GPU ${gpuLabel}, CUDA ${report.cuda_available ? "yes" : "no"}`,
6634
+ `Adapter: ${adapterBits.join("; ")}`,
6635
+ `Foundation: train ${report.foundation.train.status}, suggested max depth ${report.foundation.suggested_max_depth}; serve ${report.foundation.serve.status}`
6636
+ ].join(". ");
6637
+ }
6638
+ function warningsFromSnapshot(report, target) {
6639
+ const warnings = [];
6640
+ if (target.engine === "decision") {
6641
+ const decision = report.workflows?.find((item) => item.id === "decision");
6642
+ if (decision && decision.status !== "ready") warnings.push(`${decision.label}: ${WORKFLOW_STATUS_LABEL[decision.status]} \u2014 ${decision.reason} ${decision.next}`);
6643
+ return warnings;
6644
+ }
6645
+ if (target.engine === "foundation") {
6646
+ for (const kind of ["train", "finetune"]) {
6647
+ const verdict = report.foundation[kind];
6648
+ if (verdict.status === "not_possible" || verdict.status === "tight") {
6649
+ warnings.push(`Foundation ${kind} is ${verdict.status}: ${verdict.reason}`);
6650
+ }
6651
+ }
6652
+ return warnings;
6653
+ }
6654
+ const modelId = target.baseModel;
6655
+ const adapter = modelId ? report.adapters.find((item) => item.id.toLowerCase() === modelId.trim().toLowerCase()) : void 0;
6656
+ if (!adapter) {
6657
+ if (modelId && !adapterMemoryProfile(modelId)) return warnings;
6658
+ const blocked = report.adapters.filter((item) => item.train.status === "not_possible");
6659
+ if (blocked.length === report.adapters.length) {
6660
+ warnings.push(`Adapter train is not_possible: ${report.adapters[0]?.train.reason ?? "CUDA required"}`);
6661
+ }
6662
+ return warnings;
6663
+ }
6664
+ for (const kind of ["train", "finetune", "inference"]) {
6665
+ const verdict = adapter[kind];
6666
+ if (verdict.status === "not_possible" || verdict.status === "tight") {
6667
+ warnings.push(`${adapter.id} ${kind} is ${verdict.status}: ${verdict.reason}`);
6668
+ }
6669
+ }
6670
+ return warnings;
6671
+ }
6672
+
5294
6673
  // src/local-runtime/hardware-snapshot.ts
5295
- import { chmod as chmod2, mkdir as mkdir9, readFile as readFile10, writeFile as writeFile7 } from "fs/promises";
5296
- import { dirname as dirname8, join as join10 } from "path";
6674
+ import { chmod as chmod2, mkdir as mkdir11, readFile as readFile10, writeFile as writeFile9 } from "fs/promises";
6675
+ import { dirname as dirname11, join as join15 } from "path";
5297
6676
  var HARDWARE_SNAPSHOT_VERSION = 1;
5298
6677
  var HARDWARE_SNAPSHOT_TTL_MS = 24 * 60 * 60 * 1e3;
5299
6678
  function hardwareSnapshotPath(env = process.env) {
5300
- return join10(getTunedTensorHome(env), "hardware.json");
6679
+ return join15(getTunedTensorHome(env), "hardware.json");
5301
6680
  }
5302
6681
  async function readHardwareSnapshot(env = process.env) {
5303
6682
  const path = hardwareSnapshotPath(env);
@@ -5311,7 +6690,11 @@ async function readHardwareSnapshot(env = process.env) {
5311
6690
  }
5312
6691
  const age = Date.now() - Date.parse(snapshot.collected_at);
5313
6692
  const stale = !Number.isFinite(age) || age > HARDWARE_SNAPSHOT_TTL_MS || age < 0;
5314
- return { ...snapshot, stale, path };
6693
+ if (!snapshot.inventory.os || !snapshot.inventory.node || !snapshot.inventory.nvidia_smi || !Array.isArray(snapshot.inventory.gpus) || !Array.isArray(snapshot.inventory.disks)) {
6694
+ return { ...snapshot, stale, path };
6695
+ }
6696
+ const capabilities = evaluateCapabilities(snapshot.inventory);
6697
+ return { ...snapshot, capabilities, summary: formatCapabilitySummary(capabilities), stale, path };
5315
6698
  } catch (error) {
5316
6699
  if (error.code === "ENOENT") return void 0;
5317
6700
  return void 0;
@@ -5319,7 +6702,7 @@ async function readHardwareSnapshot(env = process.env) {
5319
6702
  }
5320
6703
  async function writeHardwareSnapshot(snapshot, env = process.env) {
5321
6704
  const path = hardwareSnapshotPath(env);
5322
- await mkdir9(dirname8(path), { recursive: true, mode: 448 });
6705
+ await mkdir11(dirname11(path), { recursive: true, mode: 448 });
5323
6706
  const gpu = snapshot.capabilities.gpu;
5324
6707
  const record = {
5325
6708
  version: HARDWARE_SNAPSHOT_VERSION,
@@ -5330,7 +6713,7 @@ async function writeHardwareSnapshot(snapshot, env = process.env) {
5330
6713
  capabilities: snapshot.capabilities,
5331
6714
  summary: snapshot.summary ?? formatCapabilitySummary(snapshot.capabilities)
5332
6715
  };
5333
- await writeFile7(path, `${JSON.stringify(record, null, 2)}
6716
+ await writeFile9(path, `${JSON.stringify(record, null, 2)}
5334
6717
  `, { mode: 384 });
5335
6718
  await chmod2(path, 384);
5336
6719
  return record;
@@ -5477,8 +6860,32 @@ export {
5477
6860
  canonicalJson2 as canonicalJson,
5478
6861
  isFoundationPipeline2 as isFoundationPipeline,
5479
6862
  parsePipeline2 as parsePipeline,
6863
+ LocalProjectSpecSchema,
6864
+ isKnownLocalSpecFailure,
6865
+ canonicalWorkspace,
6866
+ fingerprintWorkspace,
6867
+ prepareLocalSpecProject,
6868
+ createLocalSpecProject,
6869
+ inspectLocalSpec,
6870
+ specDiff,
6871
+ prepareSpecUpdate,
6872
+ SpecWorkspaceMismatchError,
6873
+ validateSpecUpdate,
6874
+ applySpecUpdate,
6875
+ readSpecHistory,
6876
+ listLocalDatasets,
6877
+ inspectLocalDataset,
6878
+ inspectDatasetFile,
6879
+ prepareDatasetSplit,
6880
+ validateDatasetSplit,
6881
+ executeDatasetSplit,
5480
6882
  checkAwsGpu,
5481
6883
  runGpuProcess,
6884
+ chainEvent,
6885
+ verifyEventChain,
6886
+ withAppendLock,
6887
+ readJsonlTail,
6888
+ parseJsonlTolerant,
5482
6889
  auditRun,
5483
6890
  isTerminalRunState,
5484
6891
  defaultLocalHome,
@@ -5508,6 +6915,8 @@ export {
5508
6915
  configForSpec,
5509
6916
  resolveDownloadConfig,
5510
6917
  resolveProjectConfig,
6918
+ sampleLiveUsage,
6919
+ WORKFLOW_STATUS_LABEL,
5511
6920
  evaluateCapabilities,
5512
6921
  warningsFromSnapshot,
5513
6922
  readHardwareSnapshot,
@@ -5516,4 +6925,4 @@ export {
5516
6925
  assessHardware,
5517
6926
  createRunConsoleReporter
5518
6927
  };
5519
- //# sourceMappingURL=chunk-7RJGLRMO.js.map
6928
+ //# sourceMappingURL=chunk-FLRRDP3M.js.map