@tuned-tensor/cli 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -5
- package/dist/{chunk-RPWOCVIL.js → chunk-7RJGLRMO.js} +750 -3131
- package/dist/chunk-7RJGLRMO.js.map +1 -0
- package/dist/chunk-UCJCWJ6F.js +2809 -0
- package/dist/chunk-UCJCWJ6F.js.map +1 -0
- package/dist/index.js +1148 -399
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +183 -71
- package/dist/local-runtime.js.map +1 -1
- package/dist/prefetch-GP3OPUUL.js +18 -0
- package/dist/prefetch-GP3OPUUL.js.map +1 -0
- package/docs/local-runtime/architecture.md +1 -1
- package/docs/spec-workflow.md +17 -3
- package/package.json +4 -1
- package/training/adapter/src/prefetch.py +11 -5
- package/dist/chunk-RPWOCVIL.js.map +0 -1
|
@@ -0,0 +1,2809 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/local-runtime/prefetch.ts
|
|
4
|
+
import { createHash as createHash4 } from "crypto";
|
|
5
|
+
import { createReadStream as createReadStream3 } from "fs";
|
|
6
|
+
import { lstat as lstat3, mkdir as mkdir3, readFile as readFile3, readdir as readdir4, realpath as realpath3, stat as stat3 } from "fs/promises";
|
|
7
|
+
import { isAbsolute as isAbsolute3, join as join6, relative as relative3, resolve as resolve7, sep as sep3 } from "path";
|
|
8
|
+
|
|
9
|
+
// src/local-runtime/contracts.ts
|
|
10
|
+
import { z } from "zod";
|
|
11
|
+
import { pipelineDocumentSchema, isFoundationPipeline, parsePipeline } from "@tuned-tensor/pipeline-contract";
|
|
12
|
+
|
|
13
|
+
// src/paths.ts
|
|
14
|
+
import { existsSync } from "fs";
|
|
15
|
+
import { homedir } from "os";
|
|
16
|
+
import { isAbsolute, join, resolve } from "path";
|
|
17
|
+
var TUNED_TENSOR_DIR = ".tuned-tensor";
|
|
18
|
+
var DEFAULT_ARTIFACT_ROOT = `${TUNED_TENSOR_DIR}/artifacts`;
|
|
19
|
+
var DEFAULT_PROJECT_STORE_ROOT = `${TUNED_TENSOR_DIR}/store`;
|
|
20
|
+
var DEFAULT_FOUNDATION_RUNS_DIR = `${TUNED_TENSOR_DIR}/foundation-runs`;
|
|
21
|
+
var DEFAULT_DECISION_RUNS_DIR = `${TUNED_TENSOR_DIR}/decision-runs`;
|
|
22
|
+
function envHome(env = process.env) {
|
|
23
|
+
const home = env.HOME?.trim();
|
|
24
|
+
return home ? resolve(home) : homedir();
|
|
25
|
+
}
|
|
26
|
+
function expandUserPath(value, env = process.env, baseDirectory = process.cwd()) {
|
|
27
|
+
const trimmed = value.trim();
|
|
28
|
+
if (trimmed === "~") return envHome(env);
|
|
29
|
+
if (trimmed.startsWith("~/")) return resolve(envHome(env), trimmed.slice(2));
|
|
30
|
+
return isAbsolute(trimmed) ? resolve(trimmed) : resolve(baseDirectory, trimmed);
|
|
31
|
+
}
|
|
32
|
+
function getTunedTensorHome(env = process.env) {
|
|
33
|
+
const override = env.TUNED_TENSOR_HOME?.trim();
|
|
34
|
+
if (override) return expandUserPath(override, env);
|
|
35
|
+
return join(envHome(env), TUNED_TENSOR_DIR);
|
|
36
|
+
}
|
|
37
|
+
function legacyConfigDir(env) {
|
|
38
|
+
const configHome = env.XDG_CONFIG_HOME?.trim() ? resolve(env.XDG_CONFIG_HOME) : join(envHome(env), ".config");
|
|
39
|
+
return join(configHome, "tuned-tensor");
|
|
40
|
+
}
|
|
41
|
+
function getConfigDir(env = process.env) {
|
|
42
|
+
const next = getTunedTensorHome(env);
|
|
43
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
44
|
+
const legacy = legacyConfigDir(env);
|
|
45
|
+
if (!existsSync(join(next, "config.json")) && existsSync(join(legacy, "config.json"))) {
|
|
46
|
+
return legacy;
|
|
47
|
+
}
|
|
48
|
+
return next;
|
|
49
|
+
}
|
|
50
|
+
function getAgentConfigDir(env = process.env) {
|
|
51
|
+
const next = join(getTunedTensorHome(env), "agent");
|
|
52
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
53
|
+
const legacy = join(legacyConfigDir(env), "agent");
|
|
54
|
+
if (!existsSync(next) && existsSync(legacy)) return legacy;
|
|
55
|
+
return next;
|
|
56
|
+
}
|
|
57
|
+
function defaultStoreRoot(env = process.env) {
|
|
58
|
+
const override = env.TT_LOCAL_HOME?.trim();
|
|
59
|
+
if (override) return expandUserPath(override, env);
|
|
60
|
+
const next = join(getTunedTensorHome(env), "store");
|
|
61
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
62
|
+
const legacy = join(envHome(env), ".tuned-tensor-local");
|
|
63
|
+
if (!existsSync(next) && existsSync(legacy)) return legacy;
|
|
64
|
+
return next;
|
|
65
|
+
}
|
|
66
|
+
function defaultCacheRoot(env = process.env) {
|
|
67
|
+
return join(getTunedTensorHome(env), "cache");
|
|
68
|
+
}
|
|
69
|
+
function pythonEnvironmentPath(kind, hash, env = process.env) {
|
|
70
|
+
const next = join(defaultCacheRoot(env), kind, hash);
|
|
71
|
+
if (env.TUNED_TENSOR_HOME?.trim()) return next;
|
|
72
|
+
const cacheHome = env.XDG_CACHE_HOME?.trim() ? resolve(env.XDG_CACHE_HOME) : join(envHome(env), ".cache");
|
|
73
|
+
const legacy = join(cacheHome, "tuned-tensor-local", kind, hash);
|
|
74
|
+
if (!existsSync(next) && existsSync(legacy)) return legacy;
|
|
75
|
+
return next;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// src/local-runtime/model-registry.ts
|
|
79
|
+
import { lstat as lstat2, readdir as readdir2 } from "fs/promises";
|
|
80
|
+
import { resolve as resolve3 } from "path";
|
|
81
|
+
import { fileURLToPath } from "url";
|
|
82
|
+
|
|
83
|
+
// src/local-runtime/artifacts.ts
|
|
84
|
+
import { createHash, randomUUID } from "crypto";
|
|
85
|
+
import { createReadStream } from "fs";
|
|
86
|
+
import { copyFile, lstat, mkdir, open, readFile, readdir, realpath, rename, stat, writeFile } from "fs/promises";
|
|
87
|
+
import { dirname, isAbsolute as isAbsolute2, join as join2, relative, resolve as resolve2, sep } from "path";
|
|
88
|
+
import { createGunzip } from "zlib";
|
|
89
|
+
var RUN_OWNER_FILE = ".tt-local-run-owner.json";
|
|
90
|
+
var ARTIFACT_WORKFLOW_LOCK_FILE = ".tt-local-workflow.lock";
|
|
91
|
+
function fileUri(path) {
|
|
92
|
+
return `file://${resolve2(path)}`;
|
|
93
|
+
}
|
|
94
|
+
function defaultArtifactPrefix(input) {
|
|
95
|
+
return join2("users", input.userId, "specs", input.behaviorSpecId, "runs", input.runId);
|
|
96
|
+
}
|
|
97
|
+
function resolveRunArtifacts(args) {
|
|
98
|
+
const root = resolve2(args.artifactRoot);
|
|
99
|
+
const safePrefix = args.prefix.replace(/^[/\\]+/, "");
|
|
100
|
+
const runDir = resolve2(root, safePrefix);
|
|
101
|
+
const containment = relative(root, runDir);
|
|
102
|
+
if (containment === ".." || containment.startsWith(`..${sep}`) || isAbsolute2(containment)) {
|
|
103
|
+
throw new Error(`Artifact prefix escapes artifactRoot: ${args.prefix}`);
|
|
104
|
+
}
|
|
105
|
+
const trainingDir = join2(runDir, "training");
|
|
106
|
+
return {
|
|
107
|
+
root,
|
|
108
|
+
prefix: safePrefix,
|
|
109
|
+
runDir,
|
|
110
|
+
trainingJsonl: join2(runDir, "training.jsonl"),
|
|
111
|
+
stageMetadataJson: join2(runDir, "stage-metadata.json"),
|
|
112
|
+
trainingReportJson: join2(runDir, "training-report.json"),
|
|
113
|
+
baselineEvalJson: join2(runDir, "baseline-eval.json"),
|
|
114
|
+
candidateEvalJson: join2(runDir, "candidate-eval.json"),
|
|
115
|
+
generalBaselineEvalJson: join2(runDir, "general-baseline-eval.json"),
|
|
116
|
+
generalCandidateEvalJson: join2(runDir, "general-candidate-eval.json"),
|
|
117
|
+
runReportJson: join2(runDir, "run-report.json"),
|
|
118
|
+
artifactManifestJson: join2(runDir, "artifact-manifest.json"),
|
|
119
|
+
progressJsonl: join2(runDir, "progress.jsonl"),
|
|
120
|
+
trainingDir,
|
|
121
|
+
trainingInputDir: join2(trainingDir, "input", "data", "training"),
|
|
122
|
+
trainingConfigDir: join2(trainingDir, "input", "config"),
|
|
123
|
+
trainingOutputDir: join2(trainingDir, "output"),
|
|
124
|
+
trainingModelDir: join2(trainingDir, "model"),
|
|
125
|
+
trainingLog: join2(trainingDir, "training.log")
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
async function ensureSafeRunDirectory(artifacts) {
|
|
129
|
+
await mkdir(artifacts.root, { recursive: true });
|
|
130
|
+
const canonicalRoot = await realpath(artifacts.root);
|
|
131
|
+
const pathFromRoot = relative(artifacts.root, artifacts.runDir);
|
|
132
|
+
const segments = pathFromRoot ? pathFromRoot.split(sep).filter(Boolean) : [];
|
|
133
|
+
let current = artifacts.root;
|
|
134
|
+
for (const segment of segments) {
|
|
135
|
+
current = join2(current, segment);
|
|
136
|
+
let metadata = await lstat(current).catch((error) => {
|
|
137
|
+
if (error.code === "ENOENT") return null;
|
|
138
|
+
throw error;
|
|
139
|
+
});
|
|
140
|
+
if (!metadata) {
|
|
141
|
+
await mkdir(current).catch((error) => {
|
|
142
|
+
if (error.code !== "EEXIST") throw error;
|
|
143
|
+
});
|
|
144
|
+
metadata = await lstat(current);
|
|
145
|
+
}
|
|
146
|
+
if (metadata.isSymbolicLink()) {
|
|
147
|
+
throw new Error(`Artifact path must not contain symbolic links: ${current}`);
|
|
148
|
+
}
|
|
149
|
+
if (!metadata.isDirectory()) {
|
|
150
|
+
throw new Error(`Artifact path component is not a directory: ${current}`);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
const canonicalRunDir = await realpath(artifacts.runDir);
|
|
154
|
+
const containment = relative(canonicalRoot, canonicalRunDir);
|
|
155
|
+
if (containment === ".." || containment.startsWith(`..${sep}`) || isAbsolute2(containment)) {
|
|
156
|
+
throw new Error(`Artifact run directory escapes artifactRoot: ${artifacts.runDir}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
async function claimRunArtifactDirectory(args) {
|
|
160
|
+
await ensureSafeRunDirectory(args.artifacts);
|
|
161
|
+
const ownerPath = join2(args.artifacts.runDir, RUN_OWNER_FILE);
|
|
162
|
+
const proposed = {
|
|
163
|
+
schema_version: 1,
|
|
164
|
+
run_id: args.runId,
|
|
165
|
+
user_id: args.userId,
|
|
166
|
+
behavior_spec_id: args.behaviorSpecId,
|
|
167
|
+
created_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
168
|
+
};
|
|
169
|
+
try {
|
|
170
|
+
const handle = await open(ownerPath, "wx", 384);
|
|
171
|
+
try {
|
|
172
|
+
await handle.writeFile(`${JSON.stringify(proposed, null, 2)}
|
|
173
|
+
`);
|
|
174
|
+
await handle.sync();
|
|
175
|
+
} finally {
|
|
176
|
+
await handle.close();
|
|
177
|
+
}
|
|
178
|
+
return proposed;
|
|
179
|
+
} catch (error) {
|
|
180
|
+
if (error.code !== "EEXIST") throw error;
|
|
181
|
+
}
|
|
182
|
+
let existing;
|
|
183
|
+
try {
|
|
184
|
+
existing = JSON.parse(await readFile(ownerPath, "utf8"));
|
|
185
|
+
} catch (error) {
|
|
186
|
+
throw new Error(`Artifact run owner claim is unreadable: ${ownerPath}`, { cause: error });
|
|
187
|
+
}
|
|
188
|
+
if (existing.schema_version !== 1 || existing.run_id !== args.runId || existing.user_id !== args.userId || existing.behavior_spec_id !== args.behaviorSpecId) {
|
|
189
|
+
throw new Error(
|
|
190
|
+
`Artifact directory ${args.artifacts.runDir} is already owned by run ${existing.run_id ?? "unknown"}; run ${args.runId} must use a different artifacts.prefix.`
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
return existing;
|
|
194
|
+
}
|
|
195
|
+
async function prepareRunDirectories(artifacts) {
|
|
196
|
+
await ensureSafeRunDirectory(artifacts);
|
|
197
|
+
await Promise.all([
|
|
198
|
+
mkdir(artifacts.runDir, { recursive: true }),
|
|
199
|
+
mkdir(artifacts.trainingInputDir, { recursive: true }),
|
|
200
|
+
mkdir(artifacts.trainingConfigDir, { recursive: true }),
|
|
201
|
+
mkdir(artifacts.trainingOutputDir, { recursive: true }),
|
|
202
|
+
mkdir(artifacts.trainingModelDir, { recursive: true })
|
|
203
|
+
]);
|
|
204
|
+
}
|
|
205
|
+
async function writeJson(path, value) {
|
|
206
|
+
await mkdir(dirname(path), { recursive: true });
|
|
207
|
+
await writeFile(path, `${JSON.stringify(value, null, 2)}
|
|
208
|
+
`, "utf8");
|
|
209
|
+
}
|
|
210
|
+
async function writeJsonAtomic(path, value) {
|
|
211
|
+
await writeFileAtomic(path, `${JSON.stringify(value, null, 2)}
|
|
212
|
+
`);
|
|
213
|
+
}
|
|
214
|
+
async function writeFileAtomic(path, value) {
|
|
215
|
+
await mkdir(dirname(path), { recursive: true });
|
|
216
|
+
const temporary = `${path}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
|
217
|
+
await writeFile(temporary, value);
|
|
218
|
+
await rename(temporary, path);
|
|
219
|
+
}
|
|
220
|
+
async function readJson(path) {
|
|
221
|
+
return JSON.parse(await readFile(path, "utf8"));
|
|
222
|
+
}
|
|
223
|
+
async function copyDatasetToTrainingChannel(artifacts) {
|
|
224
|
+
const destination = join2(artifacts.trainingInputDir, "training.jsonl");
|
|
225
|
+
await copyFile(artifacts.trainingJsonl, destination);
|
|
226
|
+
return destination;
|
|
227
|
+
}
|
|
228
|
+
var MANIFEST_IGNORED_PATHS = /* @__PURE__ */ new Set([
|
|
229
|
+
"artifact-manifest.json",
|
|
230
|
+
// These are append-only/control-plane files rather than immutable stage
|
|
231
|
+
// outputs. Including them would invalidate a manifest whenever the store
|
|
232
|
+
// records the next stage event or refreshes the canonical request copy.
|
|
233
|
+
"progress.jsonl",
|
|
234
|
+
"request.json",
|
|
235
|
+
RUN_OWNER_FILE,
|
|
236
|
+
ARTIFACT_WORKFLOW_LOCK_FILE
|
|
237
|
+
]);
|
|
238
|
+
function portableRelative(root, path) {
|
|
239
|
+
return relative(root, path).split(sep).join("/");
|
|
240
|
+
}
|
|
241
|
+
function shouldManifest(path) {
|
|
242
|
+
const atomicTemporary = /\.\d+\.\d+\.[0-9a-f-]{36}\.tmp$/i.test(path);
|
|
243
|
+
return !MANIFEST_IGNORED_PATHS.has(path) && !atomicTemporary;
|
|
244
|
+
}
|
|
245
|
+
async function listArtifactFiles(root, current = root) {
|
|
246
|
+
const files = [];
|
|
247
|
+
for (const entry of await readdir(current, { withFileTypes: true }).catch(() => [])) {
|
|
248
|
+
const absolute = join2(current, entry.name);
|
|
249
|
+
if (entry.isSymbolicLink()) {
|
|
250
|
+
throw new Error(`Artifact tree must not contain symbolic links: ${absolute}`);
|
|
251
|
+
}
|
|
252
|
+
if (entry.isDirectory()) {
|
|
253
|
+
files.push(...await listArtifactFiles(root, absolute));
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
if (!entry.isFile()) continue;
|
|
257
|
+
const relativePath = portableRelative(root, absolute);
|
|
258
|
+
if (!shouldManifest(relativePath)) continue;
|
|
259
|
+
const metadata = await stat(absolute).catch(() => null);
|
|
260
|
+
if (metadata?.isFile()) files.push(relativePath);
|
|
261
|
+
}
|
|
262
|
+
return files.sort();
|
|
263
|
+
}
|
|
264
|
+
async function sha256File(path) {
|
|
265
|
+
const hash = createHash("sha256");
|
|
266
|
+
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
|
267
|
+
return hash.digest("hex");
|
|
268
|
+
}
|
|
269
|
+
async function describeArtifactFile(root, path) {
|
|
270
|
+
const absolute = join2(root, path);
|
|
271
|
+
const metadata = await stat(absolute);
|
|
272
|
+
return {
|
|
273
|
+
path,
|
|
274
|
+
size_bytes: metadata.size,
|
|
275
|
+
sha256: await sha256File(absolute)
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
function assertManifestFiles(value, label) {
|
|
279
|
+
if (!Array.isArray(value)) throw new Error(`Artifact manifest ${label} must be an array.`);
|
|
280
|
+
const paths = /* @__PURE__ */ new Set();
|
|
281
|
+
for (const file of value) {
|
|
282
|
+
if (!file || typeof file.path !== "string" || !Number.isSafeInteger(file.size_bytes) || file.size_bytes < 0 || typeof file.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(file.sha256)) {
|
|
283
|
+
throw new Error(`Artifact manifest contains an invalid ${label} entry.`);
|
|
284
|
+
}
|
|
285
|
+
const manifestPath2 = file.path;
|
|
286
|
+
const segments = manifestPath2.split("/");
|
|
287
|
+
if (!manifestPath2 || manifestPath2.includes("\\") || manifestPath2.includes("\0") || manifestPath2.startsWith("/") || /^[a-z]:/i.test(manifestPath2) || segments.some((segment) => !segment || segment === "." || segment === "..")) {
|
|
288
|
+
throw new Error(`Artifact manifest contains an unsafe ${label} path: ${manifestPath2}`);
|
|
289
|
+
}
|
|
290
|
+
if (paths.has(manifestPath2)) throw new Error(`Artifact manifest contains a duplicate ${label} path: ${manifestPath2}`);
|
|
291
|
+
paths.add(manifestPath2);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
function hasPeftAdapterFiles(files) {
|
|
295
|
+
const names = new Set(
|
|
296
|
+
files.filter((file) => file.size_bytes > 0).map((file) => file.path.split("/").at(-1)?.toLowerCase())
|
|
297
|
+
);
|
|
298
|
+
return names.has("adapter_config.json") && (names.has("adapter_model.safetensors") || names.has("adapter_model.bin"));
|
|
299
|
+
}
|
|
300
|
+
function parseArtifactManifest(value) {
|
|
301
|
+
if (!value || typeof value !== "object") throw new Error("Artifact manifest must be an object.");
|
|
302
|
+
const candidate = value;
|
|
303
|
+
if (candidate.schema_version !== 1 || !Array.isArray(candidate.files)) {
|
|
304
|
+
throw new Error("Unsupported or invalid artifact manifest.");
|
|
305
|
+
}
|
|
306
|
+
assertManifestFiles(candidate.files, "file");
|
|
307
|
+
if (candidate.model) {
|
|
308
|
+
const model = candidate.model;
|
|
309
|
+
const allowedModelKeys = /* @__PURE__ */ new Set([
|
|
310
|
+
"artifact_kind",
|
|
311
|
+
"format",
|
|
312
|
+
"framework",
|
|
313
|
+
"base_model",
|
|
314
|
+
"base_model_revision",
|
|
315
|
+
"base_model_artifact_uri",
|
|
316
|
+
"base_model_fingerprint",
|
|
317
|
+
"artifact_uri",
|
|
318
|
+
"artifact_root",
|
|
319
|
+
"servable",
|
|
320
|
+
"files"
|
|
321
|
+
]);
|
|
322
|
+
const hasBaseArtifact = model.base_model_artifact_uri !== void 0;
|
|
323
|
+
const hasBaseFingerprint = model.base_model_fingerprint !== void 0;
|
|
324
|
+
if (Object.keys(model).some((key) => !allowedModelKeys.has(key)) || model.artifact_kind !== "file" && model.artifact_kind !== "directory" || model.format !== "tar.gz" && model.format !== "huggingface-directory" || model.framework !== "transformers-peft" || typeof model.base_model !== "string" || model.base_model_revision !== void 0 && typeof model.base_model_revision !== "string" || model.base_model_artifact_uri !== void 0 && typeof model.base_model_artifact_uri !== "string" || model.base_model_fingerprint !== void 0 && typeof model.base_model_fingerprint !== "string" || hasBaseArtifact !== hasBaseFingerprint || typeof model.artifact_uri !== "string" || typeof model.artifact_root !== "string" || model.servable !== true || !Array.isArray(model.files) || model.files.length === 0 || model.artifact_kind === "file" && model.format !== "tar.gz" || model.artifact_kind === "directory" && model.format !== "huggingface-directory") {
|
|
325
|
+
throw new Error("Artifact manifest contains invalid model contract metadata.");
|
|
326
|
+
}
|
|
327
|
+
assertManifestFiles(model.files, "model file");
|
|
328
|
+
if (model.artifact_kind === "directory" && !hasPeftAdapterFiles(model.files)) {
|
|
329
|
+
throw new Error("PEFT model manifest requires adapter weights and adapter_config.json.");
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return candidate;
|
|
333
|
+
}
|
|
334
|
+
async function writeArtifactManifest(artifacts, options = {}) {
|
|
335
|
+
const paths = await listArtifactFiles(artifacts.runDir);
|
|
336
|
+
let model;
|
|
337
|
+
if (options.model) {
|
|
338
|
+
if (options.model.framework !== "transformers-peft" || options.model.servable !== true || options.model.artifact_kind === "file" && options.model.format !== "tar.gz" || options.model.artifact_kind === "directory" && options.model.format !== "huggingface-directory") {
|
|
339
|
+
throw new Error("TT local model artifacts must be verified, servable Transformers PEFT adapters.");
|
|
340
|
+
}
|
|
341
|
+
if (options.model.base_model_artifact_uri === void 0 !== (options.model.base_model_fingerprint === void 0)) {
|
|
342
|
+
throw new Error("Model contracts must record a local base-model URI and fingerprint together.");
|
|
343
|
+
}
|
|
344
|
+
const modelRoot = resolve2(options.model.artifact_root);
|
|
345
|
+
const modelPaths = options.model.artifact_kind === "file" ? [portableRelative(dirname(modelRoot), modelRoot)] : await listArtifactFiles(modelRoot);
|
|
346
|
+
const checksumRoot = options.model.artifact_kind === "file" ? dirname(modelRoot) : modelRoot;
|
|
347
|
+
if (options.model.artifact_kind === "file") {
|
|
348
|
+
if (!modelRoot.toLowerCase().endsWith(".tar.gz")) {
|
|
349
|
+
throw new Error(`PEFT file artifacts must be .tar.gz archives: ${modelRoot}`);
|
|
350
|
+
}
|
|
351
|
+
const archive = await verifyTarGzipArchive(modelRoot);
|
|
352
|
+
if (archive.adapter_weight_entries === 0 || archive.adapter_weight_bytes === 0) {
|
|
353
|
+
throw new Error(`PEFT model archive contains no adapter_model.safetensors or adapter_model.bin: ${modelRoot}`);
|
|
354
|
+
}
|
|
355
|
+
if (archive.adapter_config_entries === 0) {
|
|
356
|
+
throw new Error(`PEFT model archive contains no adapter_config.json: ${modelRoot}`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
const modelFiles = await Promise.all(modelPaths.map((path) => describeArtifactFile(checksumRoot, path)));
|
|
360
|
+
if (options.model.artifact_kind === "directory" && !hasPeftAdapterFiles(modelFiles)) {
|
|
361
|
+
throw new Error(`PEFT model directory requires adapter weights and adapter_config.json: ${modelRoot}`);
|
|
362
|
+
}
|
|
363
|
+
model = {
|
|
364
|
+
...options.model,
|
|
365
|
+
artifact_root: modelRoot,
|
|
366
|
+
files: modelFiles
|
|
367
|
+
};
|
|
368
|
+
}
|
|
369
|
+
const manifest = {
|
|
370
|
+
schema_version: 1,
|
|
371
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
372
|
+
files: await Promise.all(paths.map((path) => describeArtifactFile(artifacts.runDir, path))),
|
|
373
|
+
...model ? { model } : {}
|
|
374
|
+
};
|
|
375
|
+
await writeJsonAtomic(artifacts.artifactManifestJson, manifest);
|
|
376
|
+
return manifest;
|
|
377
|
+
}
|
|
378
|
+
async function verifyArtifactManifest(manifestPath2, options = {}) {
|
|
379
|
+
const root = dirname(resolve2(manifestPath2));
|
|
380
|
+
const manifest = parseArtifactManifest(JSON.parse(await readFile(manifestPath2, "utf8")));
|
|
381
|
+
const expected = new Map(manifest.files.map((file) => [file.path, file]));
|
|
382
|
+
const actualPaths = await listArtifactFiles(root);
|
|
383
|
+
const actual = new Set(actualPaths);
|
|
384
|
+
const missing = [];
|
|
385
|
+
const changed = [];
|
|
386
|
+
const normalizedRequired = (options.requiredPaths ?? []).map((path) => path.split(sep).join("/").replace(/^\.\//, ""));
|
|
387
|
+
const checkedEntries = options.scopeToRequired ? normalizedRequired.flatMap((path) => {
|
|
388
|
+
const entry = expected.get(path);
|
|
389
|
+
return entry ? [[path, entry]] : [];
|
|
390
|
+
}) : [...expected.entries()];
|
|
391
|
+
for (const [path, expectedFile] of checkedEntries) {
|
|
392
|
+
if (!actual.has(path)) {
|
|
393
|
+
missing.push(path);
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
const actualFile = await describeArtifactFile(root, path);
|
|
397
|
+
if (actualFile.size_bytes !== expectedFile.size_bytes || actualFile.sha256 !== expectedFile.sha256) {
|
|
398
|
+
changed.push(path);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
for (const normalized of normalizedRequired) {
|
|
402
|
+
if (!expected.has(normalized) && !missing.includes(normalized)) missing.push(normalized);
|
|
403
|
+
}
|
|
404
|
+
const unexpected = options.allowUnexpected || options.scopeToRequired ? [] : actualPaths.filter((path) => !expected.has(path));
|
|
405
|
+
let modelChecked = 0;
|
|
406
|
+
if (manifest.model && (options.verifyModel ?? !options.scopeToRequired)) {
|
|
407
|
+
const modelRoot = resolve2(manifest.model.artifact_root);
|
|
408
|
+
const checksumRoot = manifest.model.artifact_kind === "file" ? dirname(modelRoot) : modelRoot;
|
|
409
|
+
const modelActualPaths = manifest.model.artifact_kind === "file" ? [portableRelative(checksumRoot, modelRoot)] : await listArtifactFiles(modelRoot);
|
|
410
|
+
const modelActual = new Set(modelActualPaths);
|
|
411
|
+
for (const expectedFile of manifest.model.files) {
|
|
412
|
+
const label = `model:${expectedFile.path}`;
|
|
413
|
+
if (!modelActual.has(expectedFile.path)) {
|
|
414
|
+
missing.push(label);
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
const actualFile = await describeArtifactFile(checksumRoot, expectedFile.path);
|
|
418
|
+
modelChecked += 1;
|
|
419
|
+
if (actualFile.size_bytes !== expectedFile.size_bytes || actualFile.sha256 !== expectedFile.sha256) {
|
|
420
|
+
changed.push(label);
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
if (!options.allowUnexpected) {
|
|
424
|
+
const expectedModelPaths = new Set(manifest.model.files.map((file) => file.path));
|
|
425
|
+
unexpected.push(...modelActualPaths.filter((path) => !expectedModelPaths.has(path)).map((path) => `model:${path}`));
|
|
426
|
+
}
|
|
427
|
+
if (manifest.model.artifact_kind === "file") {
|
|
428
|
+
try {
|
|
429
|
+
const archive = await verifyTarGzipArchive(modelRoot);
|
|
430
|
+
if (archive.adapter_weight_entries === 0 || archive.adapter_weight_bytes === 0) {
|
|
431
|
+
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_adapter_weights`);
|
|
432
|
+
}
|
|
433
|
+
if (archive.adapter_config_entries === 0) {
|
|
434
|
+
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_adapter_config`);
|
|
435
|
+
}
|
|
436
|
+
} catch {
|
|
437
|
+
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:invalid_archive`);
|
|
438
|
+
}
|
|
439
|
+
} else if (!hasPeftAdapterFiles(
|
|
440
|
+
await Promise.all(modelActualPaths.map((path) => describeArtifactFile(checksumRoot, path)))
|
|
441
|
+
)) {
|
|
442
|
+
changed.push("model:missing_peft_adapter");
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
return {
|
|
446
|
+
valid: missing.length === 0 && changed.length === 0 && unexpected.length === 0,
|
|
447
|
+
checked: checkedEntries.length + modelChecked,
|
|
448
|
+
missing: [...new Set(missing)].sort(),
|
|
449
|
+
changed: changed.sort(),
|
|
450
|
+
unexpected: unexpected.sort()
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
async function verifyTarGzipArchive(path) {
|
|
454
|
+
const maxEntries = 1e5;
|
|
455
|
+
const maxExpandedBytes = 20 * 1024 * 1024 * 1024;
|
|
456
|
+
const maxPaxBytes = 1024 * 1024;
|
|
457
|
+
const safeMemberPath = (value) => {
|
|
458
|
+
const normalized = value.replaceAll("\\", "/");
|
|
459
|
+
const segments = normalized.split("/").filter(Boolean);
|
|
460
|
+
if (!normalized || normalized.startsWith("/") || /^[a-z]:\//i.test(normalized) || segments.includes("..")) {
|
|
461
|
+
throw new Error(`Unsafe tar member path ${JSON.stringify(normalized)} in ${path}.`);
|
|
462
|
+
}
|
|
463
|
+
return normalized;
|
|
464
|
+
};
|
|
465
|
+
const parsePax = (payload, global) => {
|
|
466
|
+
let offset = 0;
|
|
467
|
+
let memberPath;
|
|
468
|
+
const allowedMetadata = /* @__PURE__ */ new Set(["mtime", "atime", "ctime", "uid", "gid", "uname", "gname"]);
|
|
469
|
+
while (offset < payload.length) {
|
|
470
|
+
const separator = payload.indexOf(32, offset);
|
|
471
|
+
if (separator < 0) throw new Error(`Malformed PAX record in ${path}.`);
|
|
472
|
+
const lengthText = payload.subarray(offset, separator).toString("ascii");
|
|
473
|
+
if (!/^[1-9][0-9]*$/.test(lengthText)) throw new Error(`Malformed PAX record length in ${path}.`);
|
|
474
|
+
const length = Number.parseInt(lengthText, 10);
|
|
475
|
+
const end = offset + length;
|
|
476
|
+
if (!Number.isSafeInteger(length) || end > payload.length || payload[end - 1] !== 10) {
|
|
477
|
+
throw new Error(`Truncated PAX record in ${path}.`);
|
|
478
|
+
}
|
|
479
|
+
const record = payload.subarray(separator + 1, end - 1).toString("utf8");
|
|
480
|
+
const equals = record.indexOf("=");
|
|
481
|
+
if (equals <= 0) throw new Error(`Malformed PAX key/value in ${path}.`);
|
|
482
|
+
const key = record.slice(0, equals);
|
|
483
|
+
const value = record.slice(equals + 1);
|
|
484
|
+
if (key === "path") {
|
|
485
|
+
if (global) throw new Error(`Global PAX path overrides are unsupported in ${path}.`);
|
|
486
|
+
memberPath = safeMemberPath(value);
|
|
487
|
+
} else if (!allowedMetadata.has(key)) {
|
|
488
|
+
throw new Error(`Unsupported PAX key ${JSON.stringify(key)} in ${path}.`);
|
|
489
|
+
}
|
|
490
|
+
offset = end;
|
|
491
|
+
}
|
|
492
|
+
return memberPath;
|
|
493
|
+
};
|
|
494
|
+
const gunzip = createReadStream(path).pipe(createGunzip());
|
|
495
|
+
let buffered = Buffer.alloc(0);
|
|
496
|
+
let payloadBytes = 0;
|
|
497
|
+
let paddingBytes = 0;
|
|
498
|
+
let paxType = null;
|
|
499
|
+
let paxChunks = [];
|
|
500
|
+
let pendingPaxPath;
|
|
501
|
+
let entries = 0;
|
|
502
|
+
let payloadEntries = 0;
|
|
503
|
+
let recognizedPayloadEntries = 0;
|
|
504
|
+
let recognizedPayloadBytes = 0;
|
|
505
|
+
let adapterWeightEntries = 0;
|
|
506
|
+
let adapterWeightBytes = 0;
|
|
507
|
+
let fullModelWeightEntries = 0;
|
|
508
|
+
let fullModelWeightBytes = 0;
|
|
509
|
+
let adapterConfigEntries = 0;
|
|
510
|
+
let expandedBytes = 0;
|
|
511
|
+
let zeroBlocks = 0;
|
|
512
|
+
let ended = false;
|
|
513
|
+
for await (const chunk of gunzip) {
|
|
514
|
+
buffered = Buffer.concat([buffered, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)]);
|
|
515
|
+
while (true) {
|
|
516
|
+
if (payloadBytes > 0) {
|
|
517
|
+
if (buffered.length === 0) break;
|
|
518
|
+
const consumed = Math.min(payloadBytes, buffered.length);
|
|
519
|
+
if (paxType) paxChunks.push(buffered.subarray(0, consumed));
|
|
520
|
+
buffered = buffered.subarray(consumed);
|
|
521
|
+
payloadBytes -= consumed;
|
|
522
|
+
continue;
|
|
523
|
+
}
|
|
524
|
+
if (paddingBytes > 0) {
|
|
525
|
+
if (buffered.length === 0) break;
|
|
526
|
+
const consumed = Math.min(paddingBytes, buffered.length);
|
|
527
|
+
buffered = buffered.subarray(consumed);
|
|
528
|
+
paddingBytes -= consumed;
|
|
529
|
+
continue;
|
|
530
|
+
}
|
|
531
|
+
if (paxType) {
|
|
532
|
+
const paxPath = parsePax(Buffer.concat(paxChunks), paxType === "g");
|
|
533
|
+
if (paxType === "x") pendingPaxPath = paxPath;
|
|
534
|
+
paxType = null;
|
|
535
|
+
paxChunks = [];
|
|
536
|
+
continue;
|
|
537
|
+
}
|
|
538
|
+
if (buffered.length < 512) break;
|
|
539
|
+
const header = buffered.subarray(0, 512);
|
|
540
|
+
buffered = buffered.subarray(512);
|
|
541
|
+
if (header.every((byte) => byte === 0)) {
|
|
542
|
+
zeroBlocks += 1;
|
|
543
|
+
if (zeroBlocks >= 2) ended = true;
|
|
544
|
+
continue;
|
|
545
|
+
}
|
|
546
|
+
if (ended) throw new Error(`Tar archive contains data after its end marker: ${path}.`);
|
|
547
|
+
zeroBlocks = 0;
|
|
548
|
+
const checksumText = header.subarray(148, 156).toString("ascii").replace(/\0/g, "").trim();
|
|
549
|
+
if (!/^[0-7]+$/.test(checksumText)) throw new Error(`Invalid tar checksum field in ${path}.`);
|
|
550
|
+
const expectedChecksum = Number.parseInt(checksumText, 8);
|
|
551
|
+
let actualChecksum = 0;
|
|
552
|
+
for (let index = 0; index < header.length; index += 1) {
|
|
553
|
+
actualChecksum += index >= 148 && index < 156 ? 32 : header[index];
|
|
554
|
+
}
|
|
555
|
+
if (actualChecksum !== expectedChecksum) throw new Error(`Tar member checksum mismatch in ${path}.`);
|
|
556
|
+
const sizeText = header.subarray(124, 136).toString("ascii").replace(/\0/g, "").trim();
|
|
557
|
+
if (sizeText && !/^[0-7]+$/.test(sizeText)) throw new Error(`Invalid tar member size in ${path}.`);
|
|
558
|
+
const size = sizeText ? Number.parseInt(sizeText, 8) : 0;
|
|
559
|
+
if (!Number.isSafeInteger(size) || size < 0) throw new Error(`Invalid tar member size in ${path}.`);
|
|
560
|
+
const type = String.fromCharCode(header[156] ?? 0);
|
|
561
|
+
if (type !== "\0" && type !== "0" && type !== "5" && type !== "x" && type !== "g") {
|
|
562
|
+
throw new Error(`Unsafe or unsupported tar member type ${JSON.stringify(type)} in ${path}.`);
|
|
563
|
+
}
|
|
564
|
+
const shortName = header.subarray(0, 100).toString("utf8").replace(/\0.*$/, "");
|
|
565
|
+
const prefix = header.subarray(345, 500).toString("utf8").replace(/\0.*$/, "");
|
|
566
|
+
const headerPath = safeMemberPath(prefix ? `${prefix}/${shortName}` : shortName);
|
|
567
|
+
const memberPath = type === "x" || type === "g" ? headerPath : safeMemberPath(pendingPaxPath ?? headerPath);
|
|
568
|
+
if (type !== "x" && type !== "g") pendingPaxPath = void 0;
|
|
569
|
+
const segments = memberPath.split("/").filter(Boolean);
|
|
570
|
+
const name = segments.at(-1)?.toLowerCase();
|
|
571
|
+
const metadataOnly = name === "training-metrics.json" || name === "trainer_state.json" || name === "training_args.bin" || name === "config.json" || name === "generation_config.json" || name === "adapter_config.json" || name === "readme.md" || name?.startsWith("tokenizer") || name?.startsWith("special_tokens") || name?.startsWith("added_tokens") || name === "vocab.json" || name === "merges.txt";
|
|
572
|
+
const recognizedPayload = Boolean(name) && name !== "training_args.bin" && (name.endsWith(".safetensors") || name.endsWith(".bin"));
|
|
573
|
+
const adapterWeight = name === "adapter_model.safetensors" || name === "adapter_model.bin";
|
|
574
|
+
const fullModelWeight = Boolean(name) && (name.endsWith(".safetensors") && !name.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(name));
|
|
575
|
+
if (size > 0 && name === "adapter_config.json" && (type === "\0" || type === "0")) {
|
|
576
|
+
adapterConfigEntries += 1;
|
|
577
|
+
}
|
|
578
|
+
if ((type === "x" || type === "g") && size > maxPaxBytes) {
|
|
579
|
+
throw new Error(`PAX metadata exceeds 1 MiB in ${path}.`);
|
|
580
|
+
}
|
|
581
|
+
if (size > 0 && !metadataOnly && (type === "\0" || type === "0")) payloadEntries += 1;
|
|
582
|
+
if (size > 0 && recognizedPayload && (type === "\0" || type === "0")) {
|
|
583
|
+
recognizedPayloadEntries += 1;
|
|
584
|
+
recognizedPayloadBytes += size;
|
|
585
|
+
}
|
|
586
|
+
if (size > 0 && adapterWeight && (type === "\0" || type === "0")) {
|
|
587
|
+
adapterWeightEntries += 1;
|
|
588
|
+
adapterWeightBytes += size;
|
|
589
|
+
}
|
|
590
|
+
if (size > 0 && fullModelWeight && (type === "\0" || type === "0")) {
|
|
591
|
+
fullModelWeightEntries += 1;
|
|
592
|
+
fullModelWeightBytes += size;
|
|
593
|
+
}
|
|
594
|
+
expandedBytes += size;
|
|
595
|
+
if (expandedBytes > maxExpandedBytes) throw new Error(`Tar archive exceeds 20 GiB expanded size: ${path}.`);
|
|
596
|
+
payloadBytes = size;
|
|
597
|
+
paddingBytes = (512 - size % 512) % 512;
|
|
598
|
+
paxType = type === "x" || type === "g" ? type : null;
|
|
599
|
+
paxChunks = [];
|
|
600
|
+
entries += 1;
|
|
601
|
+
if (entries > maxEntries) throw new Error(`Tar archive exceeds ${maxEntries} members: ${path}.`);
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
if (payloadBytes !== 0 || paddingBytes !== 0 || paxType || buffered.some((byte) => byte !== 0)) {
|
|
605
|
+
throw new Error(`Truncated tar archive: ${path}.`);
|
|
606
|
+
}
|
|
607
|
+
if (pendingPaxPath) throw new Error(`PAX path override has no following member in ${path}.`);
|
|
608
|
+
if (!ended) throw new Error(`Tar archive is missing its end marker: ${path}.`);
|
|
609
|
+
if (entries === 0) throw new Error(`Empty tar archive: ${path}.`);
|
|
610
|
+
if (payloadEntries === 0) throw new Error(`Tar archive contains no model payload files: ${path}.`);
|
|
611
|
+
return {
|
|
612
|
+
entries,
|
|
613
|
+
payload_entries: payloadEntries,
|
|
614
|
+
recognized_payload_entries: recognizedPayloadEntries,
|
|
615
|
+
recognized_payload_bytes: recognizedPayloadBytes,
|
|
616
|
+
adapter_weight_entries: adapterWeightEntries,
|
|
617
|
+
adapter_weight_bytes: adapterWeightBytes,
|
|
618
|
+
full_model_weight_entries: fullModelWeightEntries,
|
|
619
|
+
full_model_weight_bytes: fullModelWeightBytes,
|
|
620
|
+
adapter_config_entries: adapterConfigEntries,
|
|
621
|
+
expanded_bytes: expandedBytes
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
async function assertArtifactManifest(manifestPath2, options = {}) {
|
|
625
|
+
const result = await verifyArtifactManifest(manifestPath2, options);
|
|
626
|
+
if (!result.valid) {
|
|
627
|
+
const details = [
|
|
628
|
+
result.missing.length ? `missing=${result.missing.join(",")}` : "",
|
|
629
|
+
result.changed.length ? `changed=${result.changed.join(",")}` : "",
|
|
630
|
+
result.unexpected.length ? `unexpected=${result.unexpected.join(",")}` : ""
|
|
631
|
+
].filter(Boolean).join(" ");
|
|
632
|
+
throw new Error(`Artifact integrity verification failed${details ? `: ${details}` : "."}`);
|
|
633
|
+
}
|
|
634
|
+
return result;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
// src/local-runtime/model-registry.ts
|
|
638
|
+
var QWEN_3_5_2B_REVISION = "15852e8c16360a2fea060d615a32b45270f8a8fc";
|
|
639
|
+
var NEMOTRON_BF16_REVISION = "ce38b6ab8b252b4b8ee7165b4605e93191cafd73";
|
|
640
|
+
var MUSE_GLIMMER_BF16_REVISION = "a4e59da52a7bc87ae7251dd5545c0dd437c44b68";
|
|
641
|
+
var TRAINING_MODELS = [
|
|
642
|
+
{
|
|
643
|
+
id: "Qwen/Qwen3.5-2B",
|
|
644
|
+
family: "qwen3_5",
|
|
645
|
+
defaultRevision: QWEN_3_5_2B_REVISION,
|
|
646
|
+
modelLoader: "causal_lm",
|
|
647
|
+
defaultLearningRate: 1e-5,
|
|
648
|
+
defaultPerDeviceBatchSize: 1,
|
|
649
|
+
defaultGradientAccumulationSteps: 8,
|
|
650
|
+
defaultLoraRank: 16,
|
|
651
|
+
defaultLoraAlpha: 32,
|
|
652
|
+
defaultLoraDropout: 0.05,
|
|
653
|
+
defaultMaxSeqLength: 2048,
|
|
654
|
+
loraTargetModules: "all-linear",
|
|
655
|
+
gradientCheckpointing: false
|
|
656
|
+
},
|
|
657
|
+
{
|
|
658
|
+
id: "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16",
|
|
659
|
+
family: "nemotron_h",
|
|
660
|
+
defaultRevision: NEMOTRON_BF16_REVISION,
|
|
661
|
+
modelLoader: "causal_lm",
|
|
662
|
+
defaultLearningRate: 1e-5,
|
|
663
|
+
defaultPerDeviceBatchSize: 1,
|
|
664
|
+
defaultGradientAccumulationSteps: 8,
|
|
665
|
+
defaultLoraRank: 16,
|
|
666
|
+
defaultLoraAlpha: 32,
|
|
667
|
+
defaultLoraDropout: 0.05,
|
|
668
|
+
defaultMaxSeqLength: 1024,
|
|
669
|
+
// Avoid one LoRA pair for every routed expert matrix. Adapt the shared
|
|
670
|
+
// attention (q/k/v/o) and Mamba input projections instead, keeping the
|
|
671
|
+
// adapter bounded. out_proj/conv1d are rejected by PEFT for Mamba models.
|
|
672
|
+
loraTargetModules: ["q_proj", "k_proj", "v_proj", "o_proj", "in_proj"],
|
|
673
|
+
gradientCheckpointing: true
|
|
674
|
+
},
|
|
675
|
+
{
|
|
676
|
+
id: "meta-models/Muse-Glimmer-30B",
|
|
677
|
+
family: "muse_glimmer",
|
|
678
|
+
defaultRevision: MUSE_GLIMMER_BF16_REVISION,
|
|
679
|
+
// Muse Glimmer is a vision-language model whose text tower is not exposed
|
|
680
|
+
// through the causal-LM auto mapping, so the bundled runtime loads the full
|
|
681
|
+
// conditional-generation checkpoint (vision tower frozen and unused) via
|
|
682
|
+
// AutoModelForImageTextToText for text-only SFT.
|
|
683
|
+
modelLoader: "image_text_to_text",
|
|
684
|
+
defaultLearningRate: 1e-5,
|
|
685
|
+
defaultPerDeviceBatchSize: 1,
|
|
686
|
+
defaultGradientAccumulationSteps: 8,
|
|
687
|
+
defaultLoraRank: 16,
|
|
688
|
+
defaultLoraAlpha: 32,
|
|
689
|
+
defaultLoraDropout: 0.05,
|
|
690
|
+
defaultMaxSeqLength: 2048,
|
|
691
|
+
loraTargetModules: "all-linear",
|
|
692
|
+
gradientCheckpointing: true
|
|
693
|
+
}
|
|
694
|
+
];
|
|
695
|
+
var HUGGING_FACE_REPO_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,95}\/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/;
|
|
696
|
+
function uncertifiedTrainingModel(id) {
|
|
697
|
+
return {
|
|
698
|
+
id,
|
|
699
|
+
family: "uncertified",
|
|
700
|
+
modelLoader: "causal_lm",
|
|
701
|
+
defaultLearningRate: 1e-4,
|
|
702
|
+
defaultPerDeviceBatchSize: 1,
|
|
703
|
+
defaultGradientAccumulationSteps: 8,
|
|
704
|
+
defaultLoraRank: 16,
|
|
705
|
+
defaultLoraAlpha: 32,
|
|
706
|
+
defaultLoraDropout: 0.05,
|
|
707
|
+
defaultMaxSeqLength: 2048,
|
|
708
|
+
loraTargetModules: "all-linear",
|
|
709
|
+
gradientCheckpointing: true
|
|
710
|
+
};
|
|
711
|
+
}
|
|
712
|
+
function findCertifiedTrainingModel(modelId) {
|
|
713
|
+
const normalized = modelId.trim().toLowerCase();
|
|
714
|
+
return TRAINING_MODELS.find((candidate) => candidate.id.toLowerCase() === normalized);
|
|
715
|
+
}
|
|
716
|
+
function isCertifiedTrainingModel(modelId) {
|
|
717
|
+
return findCertifiedTrainingModel(modelId) !== void 0;
|
|
718
|
+
}
|
|
719
|
+
function resolveTrainingModel(modelId) {
|
|
720
|
+
const model = findCertifiedTrainingModel(modelId);
|
|
721
|
+
if (model) return model;
|
|
722
|
+
const id = modelId.trim();
|
|
723
|
+
if (HUGGING_FACE_REPO_ID.test(id)) return uncertifiedTrainingModel(id);
|
|
724
|
+
throw new Error(
|
|
725
|
+
`Unsupported base model "${modelId}". Use a certified model (${TRAINING_MODELS.map((item) => item.id).join(", ")}) or a Hugging Face repo id such as owner/name.`
|
|
726
|
+
);
|
|
727
|
+
}
|
|
728
|
+
function uncertifiedRevisionError(modelId) {
|
|
729
|
+
return new Error(
|
|
730
|
+
`${modelId} is not a TT-certified base model. To fine-tune it anyway, set hyperparameters.base_model_revision to its 40-character commit SHA (see https://huggingface.co/${modelId}/commits/main). Uncertified models load as text causal LMs with safetensors weights and no remote code.`
|
|
731
|
+
);
|
|
732
|
+
}
|
|
733
|
+
function canonicalizeTrainingModel(modelId) {
|
|
734
|
+
return resolveTrainingModel(modelId).id;
|
|
735
|
+
}
|
|
736
|
+
function resolveRequestedBaseModelRevision(modelId, requestedRevision) {
|
|
737
|
+
const model = resolveTrainingModel(modelId);
|
|
738
|
+
if (!isCertifiedTrainingModel(model.id) && !requestedRevision) {
|
|
739
|
+
throw uncertifiedRevisionError(model.id);
|
|
740
|
+
}
|
|
741
|
+
if (model.defaultRevision && requestedRevision && requestedRevision !== model.defaultRevision) {
|
|
742
|
+
throw new Error(
|
|
743
|
+
`${model.id} must use certified revision ${model.defaultRevision}; got ${requestedRevision}.`
|
|
744
|
+
);
|
|
745
|
+
}
|
|
746
|
+
return requestedRevision ?? model.defaultRevision;
|
|
747
|
+
}
|
|
748
|
+
function defaultBaseModelRevision(modelId) {
|
|
749
|
+
return resolveTrainingModel(modelId).defaultRevision;
|
|
750
|
+
}
|
|
751
|
+
function resolveModelLoader(modelId) {
|
|
752
|
+
return resolveTrainingModel(modelId).modelLoader;
|
|
753
|
+
}
|
|
754
|
+
var CERTIFIED_QWEN_TEXT_CONFIG = {
|
|
755
|
+
model_type: "qwen3_5_text",
|
|
756
|
+
hidden_size: 2048,
|
|
757
|
+
num_hidden_layers: 24,
|
|
758
|
+
num_attention_heads: 8,
|
|
759
|
+
num_key_value_heads: 2,
|
|
760
|
+
intermediate_size: 6144,
|
|
761
|
+
vocab_size: 248320
|
|
762
|
+
};
|
|
763
|
+
var CERTIFIED_MUSE_GLIMMER_TEXT_CONFIG = {
|
|
764
|
+
model_type: "muse_glimmer_text",
|
|
765
|
+
hidden_size: 6656,
|
|
766
|
+
num_hidden_layers: 52,
|
|
767
|
+
num_attention_heads: 32,
|
|
768
|
+
num_key_value_heads: 2,
|
|
769
|
+
intermediate_size: 19968,
|
|
770
|
+
vocab_size: 202048
|
|
771
|
+
};
|
|
772
|
+
var CERTIFIED_NEMOTRON_CONFIG = {
|
|
773
|
+
hidden_size: 2688,
|
|
774
|
+
num_hidden_layers: 52,
|
|
775
|
+
num_attention_heads: 32,
|
|
776
|
+
num_key_value_heads: 2,
|
|
777
|
+
intermediate_size: 1856,
|
|
778
|
+
vocab_size: 131072,
|
|
779
|
+
n_routed_experts: 128,
|
|
780
|
+
num_experts_per_tok: 6,
|
|
781
|
+
num_nextn_predict_layers: 1,
|
|
782
|
+
max_position_embeddings: 262144
|
|
783
|
+
};
|
|
784
|
+
function isRecord(value) {
|
|
785
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
786
|
+
}
|
|
787
|
+
function hasArchitecture(config, architecture) {
|
|
788
|
+
const architectures = config.architectures;
|
|
789
|
+
return Array.isArray(architectures) && architectures.includes(architecture);
|
|
790
|
+
}
|
|
791
|
+
function assertCertifiedTextTowerConfig(config, label, certifiedModelId, architecture, certifiedTextConfig) {
|
|
792
|
+
if (!hasArchitecture(config, architecture)) {
|
|
793
|
+
throw new Error(`${label} is not the certified ${certifiedModelId} architecture.`);
|
|
794
|
+
}
|
|
795
|
+
const textConfig = config.text_config;
|
|
796
|
+
if (!isRecord(textConfig)) {
|
|
797
|
+
throw new Error(`${label} is not the certified ${certifiedModelId} architecture.`);
|
|
798
|
+
}
|
|
799
|
+
for (const [key, expected] of Object.entries(certifiedTextConfig)) {
|
|
800
|
+
if (textConfig[key] !== expected) {
|
|
801
|
+
throw new Error(
|
|
802
|
+
`${label} is not the certified ${certifiedModelId} architecture: text_config.${key} must be ${JSON.stringify(expected)}.`
|
|
803
|
+
);
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
function assertCertifiedBaseModelConfig(value, label = "base-model config.json", expectedModelId) {
|
|
808
|
+
if (!isRecord(value)) {
|
|
809
|
+
throw new Error(`${label} must contain a JSON object.`);
|
|
810
|
+
}
|
|
811
|
+
const config = value;
|
|
812
|
+
if (expectedModelId && !isCertifiedTrainingModel(expectedModelId)) return;
|
|
813
|
+
const expectedFamily = expectedModelId ? resolveTrainingModel(expectedModelId).family : void 0;
|
|
814
|
+
if (config.model_type === "qwen3_5") {
|
|
815
|
+
if (expectedFamily && expectedFamily !== "qwen3_5") {
|
|
816
|
+
throw new Error(`${label} does not match requested base model ${expectedModelId}.`);
|
|
817
|
+
}
|
|
818
|
+
assertCertifiedTextTowerConfig(
|
|
819
|
+
config,
|
|
820
|
+
label,
|
|
821
|
+
"Qwen/Qwen3.5-2B",
|
|
822
|
+
"Qwen3_5ForConditionalGeneration",
|
|
823
|
+
CERTIFIED_QWEN_TEXT_CONFIG
|
|
824
|
+
);
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
827
|
+
if (config.model_type === "muse_glimmer") {
|
|
828
|
+
if (expectedFamily && expectedFamily !== "muse_glimmer") {
|
|
829
|
+
throw new Error(`${label} does not match requested base model ${expectedModelId}.`);
|
|
830
|
+
}
|
|
831
|
+
assertCertifiedTextTowerConfig(
|
|
832
|
+
config,
|
|
833
|
+
label,
|
|
834
|
+
"meta-models/Muse-Glimmer-30B",
|
|
835
|
+
"MuseGlimmerForConditionalGeneration",
|
|
836
|
+
CERTIFIED_MUSE_GLIMMER_TEXT_CONFIG
|
|
837
|
+
);
|
|
838
|
+
return;
|
|
839
|
+
}
|
|
840
|
+
if (config.model_type === "nemotron_h" && hasArchitecture(config, "NemotronHForCausalLM")) {
|
|
841
|
+
if (expectedFamily && expectedFamily !== "nemotron_h") {
|
|
842
|
+
throw new Error(`${label} does not match requested base model ${expectedModelId}.`);
|
|
843
|
+
}
|
|
844
|
+
for (const [key, expected] of Object.entries(CERTIFIED_NEMOTRON_CONFIG)) {
|
|
845
|
+
if (config[key] !== expected) {
|
|
846
|
+
throw new Error(
|
|
847
|
+
`${label} is not the certified nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 architecture: ${key} must be ${JSON.stringify(expected)}.`
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
return;
|
|
852
|
+
}
|
|
853
|
+
throw new Error(
|
|
854
|
+
`${label} is not a certified TT base-model architecture.`
|
|
855
|
+
);
|
|
856
|
+
}
|
|
857
|
+
function localModelArtifactPath(uri) {
|
|
858
|
+
if (uri.startsWith("file://")) return fileURLToPath(new URL(uri));
|
|
859
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(uri)) {
|
|
860
|
+
throw new Error(`Model artifact must be a local path or file URI: ${uri}`);
|
|
861
|
+
}
|
|
862
|
+
return resolve3(uri);
|
|
863
|
+
}
|
|
864
|
+
function adapterWeightName(name) {
|
|
865
|
+
const lower = name.toLowerCase();
|
|
866
|
+
return lower === "adapter_model.safetensors" || lower === "adapter_model.bin";
|
|
867
|
+
}
|
|
868
|
+
async function inspectAdapterDirectory(path) {
|
|
869
|
+
let count = 0;
|
|
870
|
+
let bytes = 0;
|
|
871
|
+
let hasConfig = false;
|
|
872
|
+
for (const entry of await readdir2(path, { withFileTypes: true })) {
|
|
873
|
+
const child = resolve3(path, entry.name);
|
|
874
|
+
if (entry.isSymbolicLink()) {
|
|
875
|
+
throw new Error(`Model artifact must not contain symbolic links: ${child}`);
|
|
876
|
+
}
|
|
877
|
+
if (entry.isDirectory()) {
|
|
878
|
+
const nested = await inspectAdapterDirectory(child);
|
|
879
|
+
count += nested.count;
|
|
880
|
+
bytes += nested.bytes;
|
|
881
|
+
hasConfig ||= nested.hasConfig;
|
|
882
|
+
continue;
|
|
883
|
+
}
|
|
884
|
+
if (!entry.isFile()) continue;
|
|
885
|
+
const metadata = await lstat2(child);
|
|
886
|
+
if (adapterWeightName(entry.name) && metadata.size > 0) {
|
|
887
|
+
count += 1;
|
|
888
|
+
bytes += metadata.size;
|
|
889
|
+
}
|
|
890
|
+
if (entry.name.toLowerCase() === "adapter_config.json" && metadata.size > 0) {
|
|
891
|
+
hasConfig = true;
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
return { count, bytes, hasConfig };
|
|
895
|
+
}
|
|
896
|
+
async function inspectModelArtifact(uri) {
|
|
897
|
+
const path = localModelArtifactPath(uri);
|
|
898
|
+
const metadata = await lstat2(path).catch(() => null);
|
|
899
|
+
if (!metadata) throw new Error(`Model artifact does not exist: ${path}`);
|
|
900
|
+
if (metadata.isSymbolicLink()) {
|
|
901
|
+
throw new Error(`Model artifact must not be a symbolic link: ${path}`);
|
|
902
|
+
}
|
|
903
|
+
if (metadata.isDirectory()) {
|
|
904
|
+
const adapter = await inspectAdapterDirectory(path);
|
|
905
|
+
return {
|
|
906
|
+
uri,
|
|
907
|
+
path,
|
|
908
|
+
kind: "directory",
|
|
909
|
+
adapter_weight_file_count: adapter.count,
|
|
910
|
+
adapter_weight_bytes: adapter.bytes,
|
|
911
|
+
has_adapter_config: adapter.hasConfig
|
|
912
|
+
};
|
|
913
|
+
}
|
|
914
|
+
if (!metadata.isFile() || !path.toLowerCase().endsWith(".tar.gz")) {
|
|
915
|
+
throw new Error(
|
|
916
|
+
`Model artifact must be a PEFT adapter directory or .tar.gz archive: ${path}`
|
|
917
|
+
);
|
|
918
|
+
}
|
|
919
|
+
const archive = await verifyTarGzipArchive(path);
|
|
920
|
+
return {
|
|
921
|
+
uri,
|
|
922
|
+
path,
|
|
923
|
+
kind: "file",
|
|
924
|
+
adapter_weight_file_count: archive.adapter_weight_entries,
|
|
925
|
+
adapter_weight_bytes: archive.adapter_weight_bytes,
|
|
926
|
+
has_adapter_config: archive.adapter_config_entries > 0
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
async function assertUsableModelArtifact(uri) {
|
|
930
|
+
const inspection = await inspectModelArtifact(uri);
|
|
931
|
+
if (inspection.adapter_weight_file_count === 0 || inspection.adapter_weight_bytes === 0) {
|
|
932
|
+
throw new Error(
|
|
933
|
+
`PEFT model artifact ${inspection.path} contains no non-empty adapter_model weights.`
|
|
934
|
+
);
|
|
935
|
+
}
|
|
936
|
+
if (!inspection.has_adapter_config) {
|
|
937
|
+
throw new Error(
|
|
938
|
+
`PEFT model artifact ${inspection.path} contains adapter weights but no non-empty adapter_config.json.`
|
|
939
|
+
);
|
|
940
|
+
}
|
|
941
|
+
return inspection;
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
// src/local-runtime/decision-models.ts
|
|
945
|
+
var LAYA_REVISION = "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851";
|
|
946
|
+
var DECISION_MODELS = [
|
|
947
|
+
{
|
|
948
|
+
id: "convaiinnovations/laya",
|
|
949
|
+
family: "laya",
|
|
950
|
+
revision: LAYA_REVISION,
|
|
951
|
+
parameters: "421M (ModernBERT-large encoder + typed decision head)",
|
|
952
|
+
license: "Apache-2.0",
|
|
953
|
+
defaultLearningRate: 2e-5,
|
|
954
|
+
defaultBatchSize: 8,
|
|
955
|
+
defaultEpochs: 3
|
|
956
|
+
}
|
|
957
|
+
];
|
|
958
|
+
var DEFAULT_DECISION_MODEL = DECISION_MODELS[0].id;
|
|
959
|
+
function resolveDecisionModel(modelId) {
|
|
960
|
+
const normalized = modelId.trim().toLowerCase();
|
|
961
|
+
const model = DECISION_MODELS.find((candidate) => candidate.id.toLowerCase() === normalized);
|
|
962
|
+
if (!model) {
|
|
963
|
+
throw new Error(
|
|
964
|
+
`Unsupported decision model "${modelId}". Supported decision models: ${DECISION_MODELS.map((item) => item.id).join(", ")}`
|
|
965
|
+
);
|
|
966
|
+
}
|
|
967
|
+
return model;
|
|
968
|
+
}
|
|
969
|
+
function isDecisionModel(modelId) {
|
|
970
|
+
return DECISION_MODELS.some((candidate) => candidate.id.toLowerCase() === modelId.trim().toLowerCase());
|
|
971
|
+
}
|
|
972
|
+
function canonicalizeDecisionModel(modelId) {
|
|
973
|
+
return resolveDecisionModel(modelId).id;
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
// src/local-runtime/contracts.ts
|
|
977
|
+
var datasetFormatSchema = z.literal("chat_jsonl");
|
|
978
|
+
var baseModelRevisionSchema = z.string().regex(/^[0-9a-f]{40}$/i, "base_model_revision must be a 40-character Hugging Face commit SHA").transform((value) => value.toLowerCase());
|
|
979
|
+
var behaviorSpecExampleSchema = z.object({
|
|
980
|
+
input: z.string().min(1),
|
|
981
|
+
output: z.string().min(1)
|
|
982
|
+
}).strict();
|
|
983
|
+
var behaviorSpecSchema = z.object({
|
|
984
|
+
name: z.string().min(1),
|
|
985
|
+
description: z.string().default(""),
|
|
986
|
+
system_prompt: z.string().default(""),
|
|
987
|
+
guidelines: z.array(z.string()).default([]),
|
|
988
|
+
examples: z.array(behaviorSpecExampleSchema).default([]),
|
|
989
|
+
constraints: z.array(z.string()).default([])
|
|
990
|
+
}).strict();
|
|
991
|
+
var specSnapshotSchema = behaviorSpecSchema.extend({
|
|
992
|
+
base_model: z.string().transform((value) => canonicalizeTrainingModel(value))
|
|
993
|
+
});
|
|
994
|
+
var foundationHyperparametersSchema = z.object({
|
|
995
|
+
vocab_size: z.number().int().min(64).max(65536).default(256),
|
|
996
|
+
max_chars: z.number().int().min(1e3).max(1e12).default(2e4),
|
|
997
|
+
depth: z.number().int().min(1).max(64).default(2),
|
|
998
|
+
pretrain_steps: z.number().int().min(1).max(1e7).default(2),
|
|
999
|
+
finetune_steps: z.number().int().min(1).max(1e7).default(2),
|
|
1000
|
+
rl_steps: z.number().int().min(0).max(1e7).default(0),
|
|
1001
|
+
batch_size: z.number().int().min(1).max(1024).default(2),
|
|
1002
|
+
sequence_length: z.number().int().min(16).max(8192).default(64),
|
|
1003
|
+
nproc_per_node: z.number().int().min(1).max(16).default(1),
|
|
1004
|
+
corpus_path: z.string().min(1).optional(),
|
|
1005
|
+
tokenizer_max_chars: z.number().int().min(1e3).max(1e12).optional(),
|
|
1006
|
+
validation_path: z.string().min(1).optional(),
|
|
1007
|
+
checkpoint_backup_dir: z.string().min(1).optional(),
|
|
1008
|
+
seed: z.number().int().min(0).max(2147483647).optional(),
|
|
1009
|
+
learning_rate: z.number().positive().max(1).optional(),
|
|
1010
|
+
weight_decay: z.number().min(0).max(10).optional(),
|
|
1011
|
+
warmup_steps: z.number().int().min(0).max(1e7).optional(),
|
|
1012
|
+
min_lr_ratio: z.number().min(0).max(1).optional(),
|
|
1013
|
+
gradient_accumulation_steps: z.number().int().min(1).max(1024).optional(),
|
|
1014
|
+
gradient_clip: z.number().positive().max(1e3).optional(),
|
|
1015
|
+
bf16: z.boolean().optional(),
|
|
1016
|
+
checkpoint_interval_steps: z.number().int().min(1).max(1e7).optional(),
|
|
1017
|
+
checkpoint_interval_seconds: z.number().int().min(1).max(86400).optional(),
|
|
1018
|
+
keep_checkpoints: z.number().int().min(1).max(100).optional(),
|
|
1019
|
+
log_interval_steps: z.number().int().min(1).max(1e6).optional()
|
|
1020
|
+
}).strict();
|
|
1021
|
+
var huggingFaceRepoIdSchema = z.string().regex(
|
|
1022
|
+
/^[A-Za-z0-9][A-Za-z0-9._-]{0,95}\/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$/,
|
|
1023
|
+
"huggingface.repo must be a Hugging Face dataset id such as org/name"
|
|
1024
|
+
);
|
|
1025
|
+
var datasetColumnsSchema = z.union([
|
|
1026
|
+
z.object({
|
|
1027
|
+
input: z.string().min(1),
|
|
1028
|
+
output: z.string().min(1)
|
|
1029
|
+
}).strict(),
|
|
1030
|
+
/** A chat column of `{role, content}` turns; its first user turn and reply become one example. */
|
|
1031
|
+
z.object({ messages: z.string().min(1) }).strict()
|
|
1032
|
+
]);
|
|
1033
|
+
var huggingFaceDatasetSourceSchema = z.object({
|
|
1034
|
+
repo: huggingFaceRepoIdSchema,
|
|
1035
|
+
revision: baseModelRevisionSchema.optional(),
|
|
1036
|
+
/**
|
|
1037
|
+
* Field names for plain JSONL or Parquet records such as `{"text": ..., "label": ...}`,
|
|
1038
|
+
* or `{messages}` for a chat column. Each record becomes a chat row with the
|
|
1039
|
+
* spec's compiled system message. Omit for files that are already TT chat JSONL.
|
|
1040
|
+
*/
|
|
1041
|
+
columns: datasetColumnsSchema.optional()
|
|
1042
|
+
}).strict();
|
|
1043
|
+
function isRepoRelativePath(value) {
|
|
1044
|
+
if (/^(?:[A-Za-z]:|[/\\]|file:)/.test(value)) return false;
|
|
1045
|
+
return value.split(/[/\\]/).every((part) => part !== "" && part !== "." && part !== "..");
|
|
1046
|
+
}
|
|
1047
|
+
var datasetPrebuiltSchema = z.object({
|
|
1048
|
+
training: z.string().min(1),
|
|
1049
|
+
validation: z.string().min(1).optional(),
|
|
1050
|
+
test: z.string().min(1).optional(),
|
|
1051
|
+
format: datasetFormatSchema.default("chat_jsonl"),
|
|
1052
|
+
huggingface: huggingFaceDatasetSourceSchema.optional()
|
|
1053
|
+
}).strict().superRefine((dataset, context) => {
|
|
1054
|
+
if (!dataset.validation && !dataset.test) {
|
|
1055
|
+
context.addIssue({
|
|
1056
|
+
code: z.ZodIssueCode.custom,
|
|
1057
|
+
path: ["validation"],
|
|
1058
|
+
message: "Provide a distinct validation or test split for held-out evaluation"
|
|
1059
|
+
});
|
|
1060
|
+
}
|
|
1061
|
+
if (!dataset.huggingface) return;
|
|
1062
|
+
for (const key of ["training", "validation", "test"]) {
|
|
1063
|
+
const value = dataset[key];
|
|
1064
|
+
if (value === void 0) continue;
|
|
1065
|
+
if (!isRepoRelativePath(value)) {
|
|
1066
|
+
context.addIssue({
|
|
1067
|
+
code: z.ZodIssueCode.custom,
|
|
1068
|
+
path: [key],
|
|
1069
|
+
message: `With dataset_prebuilt.huggingface, ${key} must be a file path inside the dataset repo, such as data/${key}.jsonl`
|
|
1070
|
+
});
|
|
1071
|
+
} else if (value.toLowerCase().endsWith(".parquet")) {
|
|
1072
|
+
if (!dataset.huggingface.columns) {
|
|
1073
|
+
context.addIssue({
|
|
1074
|
+
code: z.ZodIssueCode.custom,
|
|
1075
|
+
path: [key],
|
|
1076
|
+
message: `${key} is a Parquet file; set dataset_prebuilt.huggingface.columns to map its records`
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
} else if (!value.toLowerCase().endsWith(".jsonl")) {
|
|
1080
|
+
context.addIssue({
|
|
1081
|
+
code: z.ZodIssueCode.custom,
|
|
1082
|
+
path: [key],
|
|
1083
|
+
message: `${key} must be a .jsonl or .parquet file; CSV and other formats are not supported yet`
|
|
1084
|
+
});
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
});
|
|
1088
|
+
var datasetSourceFileSchema = z.object({
|
|
1089
|
+
path: z.string().min(1),
|
|
1090
|
+
size_bytes: z.number().int().nonnegative(),
|
|
1091
|
+
sha256: z.string().regex(/^[a-f0-9]{64}$/)
|
|
1092
|
+
}).strict();
|
|
1093
|
+
var datasetConversionStatsSchema = z.object({
|
|
1094
|
+
records: z.number().int().nonnegative(),
|
|
1095
|
+
kept: z.number().int().nonnegative(),
|
|
1096
|
+
skipped_missing_fields: z.number().int().nonnegative(),
|
|
1097
|
+
dropped_duplicate_inputs: z.number().int().nonnegative(),
|
|
1098
|
+
dropped_eval_overlap: z.number().int().nonnegative()
|
|
1099
|
+
}).strict();
|
|
1100
|
+
var datasetSourceSchema = z.object({
|
|
1101
|
+
provider: z.literal("huggingface"),
|
|
1102
|
+
repo: huggingFaceRepoIdSchema,
|
|
1103
|
+
requested_revision: baseModelRevisionSchema.nullable(),
|
|
1104
|
+
revision: baseModelRevisionSchema,
|
|
1105
|
+
pinned: z.boolean(),
|
|
1106
|
+
columns: datasetColumnsSchema.nullable(),
|
|
1107
|
+
/** Per-split record accounting when `columns` converted plain JSONL records. */
|
|
1108
|
+
conversion: z.object({
|
|
1109
|
+
training: datasetConversionStatsSchema,
|
|
1110
|
+
validation: datasetConversionStatsSchema.optional(),
|
|
1111
|
+
test: datasetConversionStatsSchema.optional()
|
|
1112
|
+
}).strict().nullable().optional(),
|
|
1113
|
+
snapshot_path: z.string().min(1),
|
|
1114
|
+
files: z.object({
|
|
1115
|
+
training: datasetSourceFileSchema,
|
|
1116
|
+
validation: datasetSourceFileSchema.optional(),
|
|
1117
|
+
test: datasetSourceFileSchema.optional()
|
|
1118
|
+
}).strict()
|
|
1119
|
+
}).strict();
|
|
1120
|
+
var fineTuneHyperparametersSchema = z.object({
|
|
1121
|
+
n_epochs: z.number().int().min(1).max(20).default(1),
|
|
1122
|
+
learning_rate: z.number().positive().optional(),
|
|
1123
|
+
batch_size: z.number().int().min(1).optional(),
|
|
1124
|
+
lora_rank: z.number().int().min(1).optional(),
|
|
1125
|
+
lora_alpha: z.number().int().min(1).optional(),
|
|
1126
|
+
lora_dropout: z.number().min(0).max(1).optional(),
|
|
1127
|
+
max_seq_length: z.number().int().min(128).max(32768).optional(),
|
|
1128
|
+
gradient_accumulation_steps: z.number().int().min(1).optional(),
|
|
1129
|
+
max_eval_examples: z.number().int().min(1).optional(),
|
|
1130
|
+
/** Immutable Hugging Face commit used by every stage of the run. */
|
|
1131
|
+
base_model_revision: baseModelRevisionSchema.optional()
|
|
1132
|
+
}).strict();
|
|
1133
|
+
function requirePinnedUncertifiedModel(spec, context) {
|
|
1134
|
+
const baseModel = "spec_snapshot" in spec ? spec.spec_snapshot.base_model : spec.base_model;
|
|
1135
|
+
if (isCertifiedTrainingModel(baseModel) || spec.hyperparameters?.base_model_revision) return;
|
|
1136
|
+
context.addIssue({
|
|
1137
|
+
code: z.ZodIssueCode.custom,
|
|
1138
|
+
path: ["hyperparameters", "base_model_revision"],
|
|
1139
|
+
message: uncertifiedRevisionError(baseModel).message
|
|
1140
|
+
});
|
|
1141
|
+
}
|
|
1142
|
+
var fineTuneRunRequestSchema = z.object({
|
|
1143
|
+
run_id: z.string().uuid(),
|
|
1144
|
+
user_id: z.string().min(1),
|
|
1145
|
+
behavior_spec_id: z.string().uuid(),
|
|
1146
|
+
run_number: z.number().int().min(1),
|
|
1147
|
+
spec_snapshot: specSnapshotSchema,
|
|
1148
|
+
hyperparameters: fineTuneHyperparametersSchema.default({ n_epochs: 1 }),
|
|
1149
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional(),
|
|
1150
|
+
dataset_source: datasetSourceSchema.optional()
|
|
1151
|
+
}).strict().superRefine((request, context) => {
|
|
1152
|
+
requirePinnedUncertifiedModel(request, context);
|
|
1153
|
+
if (request.spec_snapshot.examples.length === 0 && !request.dataset_prebuilt) {
|
|
1154
|
+
context.addIssue({
|
|
1155
|
+
code: z.ZodIssueCode.custom,
|
|
1156
|
+
path: ["spec_snapshot", "examples"],
|
|
1157
|
+
message: "Add examples or provide dataset_prebuilt training and validation files"
|
|
1158
|
+
});
|
|
1159
|
+
}
|
|
1160
|
+
});
|
|
1161
|
+
var evalExampleResultSchema = z.object({
|
|
1162
|
+
prompt: z.string(),
|
|
1163
|
+
expected: z.string(),
|
|
1164
|
+
actual: z.string(),
|
|
1165
|
+
passed: z.boolean(),
|
|
1166
|
+
score: z.number().min(0).max(1),
|
|
1167
|
+
reasoning: z.string().nullable(),
|
|
1168
|
+
latency_ms: z.number().int().nonnegative(),
|
|
1169
|
+
scored_by: z.enum(["exact_match", "json_fields", "heuristic"]).optional()
|
|
1170
|
+
}).strict();
|
|
1171
|
+
var jsonFieldMetricsSchema = z.object({
|
|
1172
|
+
fields: z.array(z.string()),
|
|
1173
|
+
valid_json_count: z.number().int().nonnegative(),
|
|
1174
|
+
valid_json_rate: z.number().min(0).max(1),
|
|
1175
|
+
schema_match_count: z.number().int().nonnegative(),
|
|
1176
|
+
schema_match_rate: z.number().min(0).max(1),
|
|
1177
|
+
all_fields_match_count: z.number().int().nonnegative(),
|
|
1178
|
+
all_fields_match_rate: z.number().min(0).max(1),
|
|
1179
|
+
field_accuracy: z.record(z.string(), z.object({
|
|
1180
|
+
correct: z.number().int().nonnegative(),
|
|
1181
|
+
total: z.number().int().nonnegative(),
|
|
1182
|
+
accuracy: z.number().min(0).max(1)
|
|
1183
|
+
}).strict())
|
|
1184
|
+
}).strict();
|
|
1185
|
+
var evalSplitSchema = z.enum([
|
|
1186
|
+
"spec_examples",
|
|
1187
|
+
"spec_holdout",
|
|
1188
|
+
"prebuilt_test",
|
|
1189
|
+
"prebuilt_validation",
|
|
1190
|
+
"general_regression"
|
|
1191
|
+
]);
|
|
1192
|
+
var evalReportSchema = z.object({
|
|
1193
|
+
kind: z.enum(["baseline", "candidate"]),
|
|
1194
|
+
model_id: z.string(),
|
|
1195
|
+
total: z.number().int().nonnegative(),
|
|
1196
|
+
eval_examples_total: z.number().int().nonnegative(),
|
|
1197
|
+
eval_examples_used: z.number().int().nonnegative(),
|
|
1198
|
+
eval_truncated: z.boolean(),
|
|
1199
|
+
eval_split: evalSplitSchema.optional(),
|
|
1200
|
+
eval_sample_seed: z.number().int().nullable().optional(),
|
|
1201
|
+
avg_score: z.number().min(0).max(1),
|
|
1202
|
+
pass_rate: z.number().min(0).max(1),
|
|
1203
|
+
exact_match_rate: z.number().min(0).max(1),
|
|
1204
|
+
avg_token_f1: z.number().min(0).max(1).optional(),
|
|
1205
|
+
avg_latency_ms: z.number().int().nonnegative(),
|
|
1206
|
+
cached: z.boolean().optional(),
|
|
1207
|
+
cache_key: z.string().optional(),
|
|
1208
|
+
results: z.array(evalExampleResultSchema),
|
|
1209
|
+
artifact_uri: z.string(),
|
|
1210
|
+
scoring_method: z.enum(["heuristic", "exact_match", "json_fields"]),
|
|
1211
|
+
inference_provider: z.enum(["none", "transformers"]).optional(),
|
|
1212
|
+
scoring_mode: z.enum(["exact_match", "json_fields"]).optional(),
|
|
1213
|
+
json_field_metrics: jsonFieldMetricsSchema.optional(),
|
|
1214
|
+
generation_config: z.record(z.string(), z.unknown()).optional(),
|
|
1215
|
+
log_uri: z.string().optional()
|
|
1216
|
+
}).strict();
|
|
1217
|
+
var comparisonReportSchema = z.object({
|
|
1218
|
+
avg_score_delta: z.number(),
|
|
1219
|
+
pass_rate_delta: z.number(),
|
|
1220
|
+
exact_match_rate_delta: z.number(),
|
|
1221
|
+
token_f1_delta: z.number().optional(),
|
|
1222
|
+
regressions: z.number().int().nonnegative(),
|
|
1223
|
+
improvements: z.number().int().nonnegative(),
|
|
1224
|
+
regressed_examples: z.array(z.object({
|
|
1225
|
+
prompt: z.string(),
|
|
1226
|
+
old_score: z.number(),
|
|
1227
|
+
new_score: z.number()
|
|
1228
|
+
}).strict())
|
|
1229
|
+
}).strict();
|
|
1230
|
+
var trainingReportSchema = z.object({
|
|
1231
|
+
provider: z.enum(["local-uv", "aws-ssh"]),
|
|
1232
|
+
training_job_name: z.string(),
|
|
1233
|
+
model_artifact_uri: z.string().optional(),
|
|
1234
|
+
base_model_artifact_uri: z.string().optional(),
|
|
1235
|
+
metrics: z.record(z.string(), z.unknown()).nullable(),
|
|
1236
|
+
exit_code: z.number().int().nullable(),
|
|
1237
|
+
log_uri: z.string(),
|
|
1238
|
+
command: z.array(z.string()).optional()
|
|
1239
|
+
}).strict();
|
|
1240
|
+
var runProvenanceSchema = z.object({
|
|
1241
|
+
schema_version: z.literal(1),
|
|
1242
|
+
recorded_at: z.string(),
|
|
1243
|
+
software: z.object({
|
|
1244
|
+
tt_version: z.string(),
|
|
1245
|
+
node_version: z.string(),
|
|
1246
|
+
platform: z.string(),
|
|
1247
|
+
arch: z.string(),
|
|
1248
|
+
runtime_fingerprint: z.string(),
|
|
1249
|
+
python_lock_sha256: z.string().nullable()
|
|
1250
|
+
}).strict(),
|
|
1251
|
+
execution: z.object({
|
|
1252
|
+
target: z.enum(["local", "aws"]),
|
|
1253
|
+
training_backend: z.string(),
|
|
1254
|
+
instance_id: z.string().optional(),
|
|
1255
|
+
region: z.string().optional()
|
|
1256
|
+
}).strict(),
|
|
1257
|
+
spec: z.object({
|
|
1258
|
+
behavior_spec_id: z.string(),
|
|
1259
|
+
run_number: z.number().int(),
|
|
1260
|
+
snapshot_sha256: z.string(),
|
|
1261
|
+
system_prompt_sha256: z.string(),
|
|
1262
|
+
file_path: z.string().optional(),
|
|
1263
|
+
file_sha256: z.string().optional()
|
|
1264
|
+
}).strict(),
|
|
1265
|
+
base_model: z.object({
|
|
1266
|
+
id: z.string(),
|
|
1267
|
+
revision: z.string().nullable(),
|
|
1268
|
+
fingerprint: z.string().nullable()
|
|
1269
|
+
}).strict(),
|
|
1270
|
+
data: z.object({
|
|
1271
|
+
prebuilt: z.boolean(),
|
|
1272
|
+
format: z.string().nullable(),
|
|
1273
|
+
file_sha256: z.record(z.string(), z.string()),
|
|
1274
|
+
compiled_training_sha256: z.string().nullable(),
|
|
1275
|
+
training_examples: z.number().int().nonnegative().nullable(),
|
|
1276
|
+
eval_split: z.string(),
|
|
1277
|
+
eval_sample_seed: z.number().int(),
|
|
1278
|
+
eval_examples_total: z.number().int().nonnegative(),
|
|
1279
|
+
eval_examples_used: z.number().int().nonnegative()
|
|
1280
|
+
}).strict(),
|
|
1281
|
+
training: z.object({
|
|
1282
|
+
hyperparameters: z.record(z.string(), z.unknown()),
|
|
1283
|
+
exit_code: z.number().int().nullable(),
|
|
1284
|
+
log_uri: z.string()
|
|
1285
|
+
}).strict(),
|
|
1286
|
+
/** Hash over request, runtime, preparation config and input digests. */
|
|
1287
|
+
source_fingerprint: z.string()
|
|
1288
|
+
}).strict();
|
|
1289
|
+
var runReportSchema = z.object({
|
|
1290
|
+
run_id: z.string().uuid(),
|
|
1291
|
+
behavior_spec_id: z.string().uuid(),
|
|
1292
|
+
user_id: z.string(),
|
|
1293
|
+
run_number: z.number().int().min(1),
|
|
1294
|
+
base_model: z.string(),
|
|
1295
|
+
fine_tuned_model_id: z.string(),
|
|
1296
|
+
status: z.enum(["completed", "failed"]),
|
|
1297
|
+
baseline: evalReportSchema,
|
|
1298
|
+
candidate: evalReportSchema,
|
|
1299
|
+
comparison: comparisonReportSchema,
|
|
1300
|
+
general_regression: z.object({
|
|
1301
|
+
dataset_uri: z.string(),
|
|
1302
|
+
dataset_sha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
1303
|
+
baseline: evalReportSchema,
|
|
1304
|
+
candidate: evalReportSchema,
|
|
1305
|
+
comparison: comparisonReportSchema,
|
|
1306
|
+
policy: z.object({
|
|
1307
|
+
max_score_drop: z.number().min(0).max(1),
|
|
1308
|
+
max_pass_rate_drop: z.number().min(0).max(1)
|
|
1309
|
+
}).strict(),
|
|
1310
|
+
passed: z.boolean(),
|
|
1311
|
+
failures: z.array(z.string())
|
|
1312
|
+
}).strict().optional(),
|
|
1313
|
+
training: trainingReportSchema,
|
|
1314
|
+
provenance: runProvenanceSchema.optional(),
|
|
1315
|
+
artifact_uris: z.object({
|
|
1316
|
+
dataset: z.string(),
|
|
1317
|
+
baseline_eval: z.string(),
|
|
1318
|
+
candidate_eval: z.string(),
|
|
1319
|
+
general_baseline_eval: z.string().optional(),
|
|
1320
|
+
general_candidate_eval: z.string().optional(),
|
|
1321
|
+
report: z.string()
|
|
1322
|
+
}).strict(),
|
|
1323
|
+
run_metadata: z.object({
|
|
1324
|
+
base_model: z.string(),
|
|
1325
|
+
fine_tuned_model_id: z.string(),
|
|
1326
|
+
dataset_prebuilt: z.boolean(),
|
|
1327
|
+
dataset_format: datasetFormatSchema.nullable().optional(),
|
|
1328
|
+
dataset_uri: z.string(),
|
|
1329
|
+
dataset_source: datasetSourceSchema.optional(),
|
|
1330
|
+
spec_example_count: z.number().int().nonnegative(),
|
|
1331
|
+
training_example_count: z.number().int().nonnegative().nullable(),
|
|
1332
|
+
eval_examples_total: z.number().int().nonnegative(),
|
|
1333
|
+
eval_examples_used: z.number().int().nonnegative(),
|
|
1334
|
+
eval_split: evalSplitSchema.optional(),
|
|
1335
|
+
eval_sample_seed: z.number().int().nullable().optional(),
|
|
1336
|
+
started_at: z.string(),
|
|
1337
|
+
completed_at: z.string(),
|
|
1338
|
+
elapsed_ms: z.number().int().nonnegative(),
|
|
1339
|
+
elapsed_seconds: z.number().nonnegative()
|
|
1340
|
+
}).strict(),
|
|
1341
|
+
created_at: z.string()
|
|
1342
|
+
}).strict();
|
|
1343
|
+
var inferenceConfigSchema = z.object({
|
|
1344
|
+
maxNewTokens: z.number().int().min(1).default(256),
|
|
1345
|
+
temperature: z.number().min(0).default(0),
|
|
1346
|
+
topP: z.number().min(0).max(1).default(1),
|
|
1347
|
+
device: z.enum(["cuda", "cpu"]).default("cuda")
|
|
1348
|
+
}).strict();
|
|
1349
|
+
var scoringConfigSchema = z.object({
|
|
1350
|
+
mode: z.enum(["exact_match", "json_fields"]).default("exact_match"),
|
|
1351
|
+
fields: z.array(z.string().min(1)).optional()
|
|
1352
|
+
}).strict();
|
|
1353
|
+
var evaluationConfigSchema = z.object({
|
|
1354
|
+
inference: inferenceConfigSchema.default({
|
|
1355
|
+
maxNewTokens: 256,
|
|
1356
|
+
temperature: 0,
|
|
1357
|
+
topP: 1,
|
|
1358
|
+
device: "cuda"
|
|
1359
|
+
}),
|
|
1360
|
+
scoring: scoringConfigSchema.default({ mode: "exact_match" }),
|
|
1361
|
+
timeoutMs: z.number().int().min(100).default(18e5),
|
|
1362
|
+
maxExamples: z.number().int().min(1).optional(),
|
|
1363
|
+
sampleSeed: z.number().int().optional(),
|
|
1364
|
+
baselineCache: z.boolean().default(true),
|
|
1365
|
+
generalRegression: z.object({
|
|
1366
|
+
dataset: z.string().min(1),
|
|
1367
|
+
systemPrompt: z.string().optional(),
|
|
1368
|
+
maxScoreDrop: z.number().min(0).max(1).default(0.03),
|
|
1369
|
+
maxPassRateDrop: z.number().min(0).max(1).default(0.05)
|
|
1370
|
+
}).strict().optional()
|
|
1371
|
+
}).strict();
|
|
1372
|
+
var localRunnerConfigSchema = z.object({
|
|
1373
|
+
gpu: z.object({
|
|
1374
|
+
provider: z.literal("aws"),
|
|
1375
|
+
instanceId: z.string().regex(/^i-[a-f0-9]{8,17}$/),
|
|
1376
|
+
profile: z.string().min(1).optional(),
|
|
1377
|
+
region: z.string().min(1).optional(),
|
|
1378
|
+
user: z.string().regex(/^[a-z_][a-z0-9_-]*$/i),
|
|
1379
|
+
identityFile: z.string().min(1).optional(),
|
|
1380
|
+
privateIp: z.boolean().default(false),
|
|
1381
|
+
maxSeconds: z.number().int().min(60).max(604800).default(86400)
|
|
1382
|
+
}).strict().optional(),
|
|
1383
|
+
storeRoot: z.string().optional(),
|
|
1384
|
+
artifactRoot: z.string().default(DEFAULT_ARTIFACT_ROOT),
|
|
1385
|
+
dryRun: z.boolean().default(false),
|
|
1386
|
+
paths: z.object({
|
|
1387
|
+
baseModel: z.string().optional(),
|
|
1388
|
+
modelCache: z.string().optional()
|
|
1389
|
+
}).strict().default({}),
|
|
1390
|
+
evaluation: evaluationConfigSchema.default({
|
|
1391
|
+
inference: {
|
|
1392
|
+
maxNewTokens: 256,
|
|
1393
|
+
temperature: 0,
|
|
1394
|
+
topP: 1,
|
|
1395
|
+
device: "cuda"
|
|
1396
|
+
},
|
|
1397
|
+
scoring: { mode: "exact_match" },
|
|
1398
|
+
timeoutMs: 18e5,
|
|
1399
|
+
baselineCache: true
|
|
1400
|
+
})
|
|
1401
|
+
}).strict();
|
|
1402
|
+
var projectRuntimeSchema = z.object({
|
|
1403
|
+
gpu: localRunnerConfigSchema.shape.gpu,
|
|
1404
|
+
artifactRoot: z.string().min(1).optional(),
|
|
1405
|
+
storeRoot: z.string().min(1).optional(),
|
|
1406
|
+
paths: z.object({ baseModel: z.string().min(1).optional(), modelCache: z.string().min(1).optional() }).strict().optional()
|
|
1407
|
+
}).strict();
|
|
1408
|
+
var projectWorkflowFields = {
|
|
1409
|
+
evaluation: evaluationConfigSchema.optional(),
|
|
1410
|
+
runtime: projectRuntimeSchema.optional(),
|
|
1411
|
+
pipeline: pipelineDocumentSchema.optional()
|
|
1412
|
+
};
|
|
1413
|
+
var localFoundationSpecFileSchema = behaviorSpecSchema.extend({
|
|
1414
|
+
...projectWorkflowFields,
|
|
1415
|
+
engine: z.literal("foundation"),
|
|
1416
|
+
id: z.string().uuid().optional(),
|
|
1417
|
+
examples: z.array(behaviorSpecExampleSchema).min(2),
|
|
1418
|
+
foundation: foundationHyperparametersSchema
|
|
1419
|
+
});
|
|
1420
|
+
var localAdapterSpecFileSchema = specSnapshotSchema.extend({
|
|
1421
|
+
...projectWorkflowFields,
|
|
1422
|
+
engine: z.literal("adapter").optional(),
|
|
1423
|
+
id: z.string().uuid().optional(),
|
|
1424
|
+
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
1425
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
1426
|
+
}).strict().superRefine(requirePinnedUncertifiedModel);
|
|
1427
|
+
var decisionQuestionSchema = z.discriminatedUnion("type", [
|
|
1428
|
+
z.object({
|
|
1429
|
+
type: z.literal("choice"),
|
|
1430
|
+
/** Answer label -> description the model reads. Key order is option order. */
|
|
1431
|
+
criteria: z.record(z.string().min(1), z.string()).refine(
|
|
1432
|
+
(criteria) => Object.keys(criteria).length >= 2 && Object.keys(criteria).length <= 32,
|
|
1433
|
+
"A choice decision needs 2 to 32 labels."
|
|
1434
|
+
)
|
|
1435
|
+
}).strict(),
|
|
1436
|
+
z.object({
|
|
1437
|
+
type: z.literal("noul"),
|
|
1438
|
+
criteria: z.object({ true: z.string().optional(), false: z.string().optional() }).strict().optional()
|
|
1439
|
+
}).strict(),
|
|
1440
|
+
z.object({
|
|
1441
|
+
type: z.literal("score"),
|
|
1442
|
+
/** Level descriptions, index 0 first; example outputs are level indices. */
|
|
1443
|
+
criteria: z.array(z.string().min(1)).min(2).max(10)
|
|
1444
|
+
}).strict()
|
|
1445
|
+
]);
|
|
1446
|
+
var decisionHyperparametersSchema = z.object({
|
|
1447
|
+
n_epochs: z.number().int().min(1).max(50).optional(),
|
|
1448
|
+
learning_rate: z.number().positive().max(1).optional(),
|
|
1449
|
+
batch_size: z.number().int().min(1).max(256).optional(),
|
|
1450
|
+
/** Train only the decision head; faster on CPU, usually less accurate. */
|
|
1451
|
+
freeze_encoder: z.boolean().optional(),
|
|
1452
|
+
/** Shuffle option order per training row so the head learns meaning, not position. */
|
|
1453
|
+
shuffle_options: z.boolean().optional(),
|
|
1454
|
+
seed: z.number().int().min(0).max(2147483647).optional(),
|
|
1455
|
+
device: z.enum(["auto", "cpu", "cuda", "mps"]).optional()
|
|
1456
|
+
}).strict();
|
|
1457
|
+
var localDecisionSpecFileSchema = behaviorSpecSchema.extend({
|
|
1458
|
+
...projectWorkflowFields,
|
|
1459
|
+
engine: z.literal("decision"),
|
|
1460
|
+
id: z.string().uuid().optional(),
|
|
1461
|
+
// Refine first: a union member must report an issue, not throw, for other engines' specs.
|
|
1462
|
+
base_model: z.string().refine(isDecisionModel, `Unsupported decision model. Supported decision models: ${DECISION_MODELS.map((model) => model.id).join(", ")}`).transform((value) => canonicalizeDecisionModel(value)),
|
|
1463
|
+
decision: decisionQuestionSchema,
|
|
1464
|
+
hyperparameters: decisionHyperparametersSchema.optional()
|
|
1465
|
+
});
|
|
1466
|
+
function decisionLabels(question) {
|
|
1467
|
+
if (question.type === "choice") return Object.keys(question.criteria);
|
|
1468
|
+
if (question.type === "noul") return ["false", "true"];
|
|
1469
|
+
return question.criteria.map((_, index) => String(index));
|
|
1470
|
+
}
|
|
1471
|
+
var localBehaviorSpecFileSchema = z.union([
|
|
1472
|
+
localFoundationSpecFileSchema,
|
|
1473
|
+
localDecisionSpecFileSchema,
|
|
1474
|
+
localAdapterSpecFileSchema
|
|
1475
|
+
]).superRefine((spec, ctx) => {
|
|
1476
|
+
if (spec.pipeline && isFoundationPipeline(parsePipeline(spec.pipeline)) !== (spec.engine === "foundation")) {
|
|
1477
|
+
ctx.addIssue({ code: "custom", path: ["pipeline"], message: "Pipeline engine must match the spec engine." });
|
|
1478
|
+
}
|
|
1479
|
+
if (spec.engine === "foundation" && (spec.evaluation || spec.runtime?.storeRoot || spec.runtime?.paths)) {
|
|
1480
|
+
ctx.addIssue({ code: "custom", path: ["runtime"], message: "Foundation uses foundation settings for evaluation and checkpoints; runtime.artifactRoot and runtime.gpu are supported." });
|
|
1481
|
+
}
|
|
1482
|
+
if (spec.engine === "decision" && (spec.evaluation || spec.runtime?.gpu || spec.runtime?.storeRoot || spec.runtime?.paths)) {
|
|
1483
|
+
ctx.addIssue({ code: "custom", path: ["runtime"], message: "Decision specs support runtime.artifactRoot only; the device is set in hyperparameters.device." });
|
|
1484
|
+
}
|
|
1485
|
+
});
|
|
1486
|
+
function isFoundationSpecFile(spec) {
|
|
1487
|
+
return spec.engine === "foundation";
|
|
1488
|
+
}
|
|
1489
|
+
function isDecisionSpecFile(spec) {
|
|
1490
|
+
return spec.engine === "decision";
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
// src/local-runtime/process-runner.ts
|
|
1494
|
+
import { createHash as createHash2 } from "crypto";
|
|
1495
|
+
import { spawn } from "child_process";
|
|
1496
|
+
import { constants, readFileSync } from "fs";
|
|
1497
|
+
import { mkdir as mkdir2, open as open2 } from "fs/promises";
|
|
1498
|
+
import { dirname as dirname3, join as join3, resolve as resolve5 } from "path";
|
|
1499
|
+
|
|
1500
|
+
// src/local-runtime/run-reporter.ts
|
|
1501
|
+
function reportInBackground(action) {
|
|
1502
|
+
try {
|
|
1503
|
+
void Promise.resolve(action()).catch(() => void 0);
|
|
1504
|
+
} catch {
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
async function withProgressHeartbeat(reporter, event, run) {
|
|
1508
|
+
if (!reporter?.onEvent) return await run();
|
|
1509
|
+
reportInBackground(() => reporter.onEvent(event));
|
|
1510
|
+
const timer = setInterval(() => {
|
|
1511
|
+
reportInBackground(() => reporter.onEvent({
|
|
1512
|
+
stage: event.stage,
|
|
1513
|
+
status: event.status,
|
|
1514
|
+
message: `Still working: ${event.message}`
|
|
1515
|
+
}));
|
|
1516
|
+
}, 15e3);
|
|
1517
|
+
timer.unref();
|
|
1518
|
+
try {
|
|
1519
|
+
return await run();
|
|
1520
|
+
} finally {
|
|
1521
|
+
clearInterval(timer);
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
function sanitizeLogLine(line) {
|
|
1525
|
+
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
1526
|
+
}
|
|
1527
|
+
function forwardStreamLines(stream, onLine) {
|
|
1528
|
+
let buffered = "";
|
|
1529
|
+
stream.setEncoding("utf8");
|
|
1530
|
+
stream.on("data", (chunk) => {
|
|
1531
|
+
buffered += chunk;
|
|
1532
|
+
const lines = buffered.split(/\r\n|\n|\r/);
|
|
1533
|
+
buffered = lines.pop() ?? "";
|
|
1534
|
+
for (const line of lines) {
|
|
1535
|
+
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
1536
|
+
}
|
|
1537
|
+
});
|
|
1538
|
+
stream.on("end", () => {
|
|
1539
|
+
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
1540
|
+
});
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
// src/local-runtime/package-root.ts
|
|
1544
|
+
import { existsSync as existsSync2 } from "fs";
|
|
1545
|
+
import { dirname as dirname2, resolve as resolve4 } from "path";
|
|
1546
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
1547
|
+
function localRuntimePackageRoot(moduleUrl) {
|
|
1548
|
+
const moduleDirectory = dirname2(fileURLToPath2(moduleUrl));
|
|
1549
|
+
const candidates = [
|
|
1550
|
+
dirname2(moduleDirectory),
|
|
1551
|
+
dirname2(dirname2(moduleDirectory))
|
|
1552
|
+
];
|
|
1553
|
+
const root = candidates.find(
|
|
1554
|
+
(candidate) => existsSync2(resolve4(candidate, "package.json")) && existsSync2(resolve4(candidate, "training/adapter"))
|
|
1555
|
+
);
|
|
1556
|
+
if (!root) {
|
|
1557
|
+
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
1558
|
+
}
|
|
1559
|
+
return root;
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1562
|
+
// src/local-runtime/progress-bar.ts
|
|
1563
|
+
var PROGRESS_PREFIX = "@@tt-progress ";
|
|
1564
|
+
function nonNegativeInteger(value) {
|
|
1565
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : null;
|
|
1566
|
+
}
|
|
1567
|
+
function parseProgressLine(line) {
|
|
1568
|
+
const start = line.indexOf(PROGRESS_PREFIX);
|
|
1569
|
+
if (start !== 0) return null;
|
|
1570
|
+
let value;
|
|
1571
|
+
try {
|
|
1572
|
+
value = JSON.parse(line.slice(PROGRESS_PREFIX.length));
|
|
1573
|
+
} catch {
|
|
1574
|
+
return null;
|
|
1575
|
+
}
|
|
1576
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
1577
|
+
const record = value;
|
|
1578
|
+
const completed = nonNegativeInteger(record.completed_bytes);
|
|
1579
|
+
const total = nonNegativeInteger(record.total_bytes);
|
|
1580
|
+
if (completed === null || total === null) return null;
|
|
1581
|
+
return {
|
|
1582
|
+
label: typeof record.label === "string" ? record.label : "download",
|
|
1583
|
+
phase: typeof record.phase === "string" ? record.phase : "download",
|
|
1584
|
+
completed_bytes: completed,
|
|
1585
|
+
total_bytes: total,
|
|
1586
|
+
files_completed: nonNegativeInteger(record.files_completed) ?? 0,
|
|
1587
|
+
files_total: nonNegativeInteger(record.files_total) ?? 0
|
|
1588
|
+
};
|
|
1589
|
+
}
|
|
1590
|
+
function formatBytes(bytes) {
|
|
1591
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
1592
|
+
let value = bytes;
|
|
1593
|
+
let unit = 0;
|
|
1594
|
+
while (value >= 1024 && unit < units.length - 1) {
|
|
1595
|
+
value /= 1024;
|
|
1596
|
+
unit += 1;
|
|
1597
|
+
}
|
|
1598
|
+
return `${unit === 0 ? value.toFixed(0) : value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 2)} ${units[unit]}`;
|
|
1599
|
+
}
|
|
1600
|
+
function formatDuration(seconds) {
|
|
1601
|
+
if (!Number.isFinite(seconds) || seconds < 0) return "--";
|
|
1602
|
+
const whole = Math.round(seconds);
|
|
1603
|
+
const hours = Math.floor(whole / 3600);
|
|
1604
|
+
const minutes = Math.floor(whole % 3600 / 60);
|
|
1605
|
+
const rest = whole % 60;
|
|
1606
|
+
if (hours > 0) return `${hours}h ${String(minutes).padStart(2, "0")}m`;
|
|
1607
|
+
if (minutes > 0) return `${minutes}m ${String(rest).padStart(2, "0")}s`;
|
|
1608
|
+
return `${rest}s`;
|
|
1609
|
+
}
|
|
1610
|
+
var RateTracker = class {
|
|
1611
|
+
constructor(windowMs = 15e3) {
|
|
1612
|
+
this.windowMs = windowMs;
|
|
1613
|
+
}
|
|
1614
|
+
samples = [];
|
|
1615
|
+
add(at, bytes) {
|
|
1616
|
+
const last = this.samples.at(-1);
|
|
1617
|
+
if (last && bytes < last.bytes) this.samples = [];
|
|
1618
|
+
this.samples.push({ at, bytes });
|
|
1619
|
+
while (this.samples.length > 2 && at - this.samples[0].at > this.windowMs) this.samples.shift();
|
|
1620
|
+
}
|
|
1621
|
+
bytesPerSecond() {
|
|
1622
|
+
const first = this.samples[0];
|
|
1623
|
+
const last = this.samples.at(-1);
|
|
1624
|
+
if (!first || !last || last.at - first.at < 500) return null;
|
|
1625
|
+
return (last.bytes - first.bytes) / ((last.at - first.at) / 1e3);
|
|
1626
|
+
}
|
|
1627
|
+
};
|
|
1628
|
+
function formatProgressLine(progress, rate, columns) {
|
|
1629
|
+
const total = progress.total_bytes;
|
|
1630
|
+
const fraction = total > 0 ? Math.min(1, progress.completed_bytes / total) : 1;
|
|
1631
|
+
const fields = [
|
|
1632
|
+
{ text: `${(fraction * 100).toFixed(1).padStart(5)}%` },
|
|
1633
|
+
{ text: `${formatBytes(progress.completed_bytes)} / ${formatBytes(total)}` }
|
|
1634
|
+
];
|
|
1635
|
+
const done = progress.completed_bytes >= total;
|
|
1636
|
+
if (!done && rate !== null && rate > 0) {
|
|
1637
|
+
fields.push(
|
|
1638
|
+
{ text: `${formatBytes(rate)}/s`, drop: 2 },
|
|
1639
|
+
{ text: `ETA ${formatDuration((total - progress.completed_bytes) / rate)}`, drop: 3 }
|
|
1640
|
+
);
|
|
1641
|
+
}
|
|
1642
|
+
if (progress.files_total > 0) fields.push({ text: `${progress.files_completed}/${progress.files_total} files`, drop: 1 });
|
|
1643
|
+
const prefix = `[tt] ${progress.label}: `;
|
|
1644
|
+
const join7 = () => fields.map((field) => field.text).join(" ");
|
|
1645
|
+
if (columns === void 0) return `${prefix}${join7()}`;
|
|
1646
|
+
for (const rank of [1, 2, 3]) {
|
|
1647
|
+
if (prefix.length + join7().length <= columns - 1) break;
|
|
1648
|
+
const index = fields.findIndex((field) => field.drop === rank);
|
|
1649
|
+
if (index !== -1) fields.splice(index, 1);
|
|
1650
|
+
}
|
|
1651
|
+
const text = join7();
|
|
1652
|
+
const barWidth = Math.min(30, columns - 1 - prefix.length - text.length - 3);
|
|
1653
|
+
if (barWidth < 10) return `${prefix}${text}`.slice(0, Math.max(0, columns - 1));
|
|
1654
|
+
const filled = Math.round(fraction * barWidth);
|
|
1655
|
+
return `${prefix}[${"#".repeat(filled)}${"-".repeat(barWidth - filled)}] ${text}`;
|
|
1656
|
+
}
|
|
1657
|
+
function createProgressRenderer(options) {
|
|
1658
|
+
const now = options.now ?? (() => Date.now());
|
|
1659
|
+
const rates = /* @__PURE__ */ new Map();
|
|
1660
|
+
const lastLine = /* @__PURE__ */ new Map();
|
|
1661
|
+
const completed = /* @__PURE__ */ new Set();
|
|
1662
|
+
let drawn = false;
|
|
1663
|
+
let lastDraw = Number.NEGATIVE_INFINITY;
|
|
1664
|
+
const finished = (progress) => progress.phase === "downloaded" || progress.total_bytes > 0 && progress.completed_bytes >= progress.total_bytes;
|
|
1665
|
+
return {
|
|
1666
|
+
update(progress) {
|
|
1667
|
+
const at = now();
|
|
1668
|
+
const tracker = rates.get(progress.label) ?? new RateTracker();
|
|
1669
|
+
rates.set(progress.label, tracker);
|
|
1670
|
+
tracker.add(at, progress.completed_bytes);
|
|
1671
|
+
const done = finished(progress);
|
|
1672
|
+
if (options.tty) {
|
|
1673
|
+
if (done && completed.has(progress.label)) return;
|
|
1674
|
+
if (!done) completed.delete(progress.label);
|
|
1675
|
+
if (!done && at - lastDraw < 100) return;
|
|
1676
|
+
lastDraw = at;
|
|
1677
|
+
const line = formatProgressLine(progress, tracker.bytesPerSecond(), Math.max(40, options.columns ?? 80));
|
|
1678
|
+
options.write(`\r\x1B[2K${line}${done ? "\n" : ""}`);
|
|
1679
|
+
drawn = !done;
|
|
1680
|
+
if (done) {
|
|
1681
|
+
rates.delete(progress.label);
|
|
1682
|
+
completed.add(progress.label);
|
|
1683
|
+
}
|
|
1684
|
+
return;
|
|
1685
|
+
}
|
|
1686
|
+
const fraction = progress.total_bytes > 0 ? progress.completed_bytes / progress.total_bytes : 1;
|
|
1687
|
+
const decile = Math.floor(fraction * 10);
|
|
1688
|
+
const previous = lastLine.get(progress.label);
|
|
1689
|
+
const interval = options.lineIntervalMs ?? 3e4;
|
|
1690
|
+
if (previous && !done && decile <= previous.decile && at - previous.at < interval) return;
|
|
1691
|
+
if (previous && done && previous.decile === 10) return;
|
|
1692
|
+
lastLine.set(progress.label, { at, decile: done ? 10 : decile });
|
|
1693
|
+
options.write(`${formatProgressLine(progress, tracker.bytesPerSecond())}
|
|
1694
|
+
`);
|
|
1695
|
+
},
|
|
1696
|
+
interrupt() {
|
|
1697
|
+
if (!drawn) return;
|
|
1698
|
+
options.write("\r\x1B[2K");
|
|
1699
|
+
drawn = false;
|
|
1700
|
+
}
|
|
1701
|
+
};
|
|
1702
|
+
}
|
|
1703
|
+
function progressTty(env = process.env, stream = process.stderr) {
|
|
1704
|
+
if (env.TT_PROGRESS_TTY === "0") return false;
|
|
1705
|
+
return env.TT_PROGRESS_TTY === "1" || Boolean(stream.isTTY);
|
|
1706
|
+
}
|
|
1707
|
+
function progressColumns(env = process.env, stream = process.stderr) {
|
|
1708
|
+
const hinted = Number(env.TT_PROGRESS_COLUMNS);
|
|
1709
|
+
if (Number.isInteger(hinted) && hinted > 0) return hinted;
|
|
1710
|
+
return stream.columns;
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
// src/local-runtime/process-runner.ts
|
|
1714
|
+
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
1715
|
+
var bundledProject = join3(packageRoot, "training/adapter");
|
|
1716
|
+
var bundledRuntimeHash = (() => {
|
|
1717
|
+
const hash = createHash2("sha256");
|
|
1718
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1719
|
+
hash.update(readFileSync(join3(bundledProject, name)));
|
|
1720
|
+
}
|
|
1721
|
+
return hash.digest("hex").slice(0, 20);
|
|
1722
|
+
})();
|
|
1723
|
+
var BUNDLED_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv", bundledRuntimeHash);
|
|
1724
|
+
var foundationProject = join3(packageRoot, "training/foundation");
|
|
1725
|
+
var foundationRuntimeHash = (() => {
|
|
1726
|
+
const hash = createHash2("sha256");
|
|
1727
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1728
|
+
hash.update(readFileSync(join3(foundationProject, name)));
|
|
1729
|
+
}
|
|
1730
|
+
return hash.digest("hex").slice(0, 20);
|
|
1731
|
+
})();
|
|
1732
|
+
var FOUNDATION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-foundation", foundationRuntimeHash);
|
|
1733
|
+
var decisionProject = join3(packageRoot, "training/decision");
|
|
1734
|
+
var decisionRuntimeHash = (() => {
|
|
1735
|
+
const hash = createHash2("sha256");
|
|
1736
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1737
|
+
hash.update(readFileSync(join3(decisionProject, name)));
|
|
1738
|
+
}
|
|
1739
|
+
return hash.digest("hex").slice(0, 20);
|
|
1740
|
+
})();
|
|
1741
|
+
var DECISION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-decision", decisionRuntimeHash);
|
|
1742
|
+
function buildDecisionPythonCommand(entrypoint, args = []) {
|
|
1743
|
+
const commandArgs = [
|
|
1744
|
+
"run",
|
|
1745
|
+
"--frozen",
|
|
1746
|
+
"--quiet",
|
|
1747
|
+
"--project",
|
|
1748
|
+
decisionProject,
|
|
1749
|
+
"python",
|
|
1750
|
+
join3(decisionProject, "src", entrypoint),
|
|
1751
|
+
...args
|
|
1752
|
+
];
|
|
1753
|
+
return { command: "uv", commandArgs, displayCommand: ["uv", ...commandArgs] };
|
|
1754
|
+
}
|
|
1755
|
+
function withDecisionPythonEnvironment(env) {
|
|
1756
|
+
return { ...env, UV_PROJECT_ENVIRONMENT: DECISION_PYTHON_ENVIRONMENT };
|
|
1757
|
+
}
|
|
1758
|
+
function buildBundledPythonCommand(entrypoint, args = []) {
|
|
1759
|
+
const target = entrypoint === "-c" ? entrypoint : join3(bundledProject, "src", entrypoint);
|
|
1760
|
+
const commandArgs = [
|
|
1761
|
+
"run",
|
|
1762
|
+
"--frozen",
|
|
1763
|
+
"--quiet",
|
|
1764
|
+
"--project",
|
|
1765
|
+
bundledProject,
|
|
1766
|
+
"python",
|
|
1767
|
+
target,
|
|
1768
|
+
...args
|
|
1769
|
+
];
|
|
1770
|
+
return {
|
|
1771
|
+
command: "uv",
|
|
1772
|
+
commandArgs,
|
|
1773
|
+
displayCommand: ["uv", ...commandArgs]
|
|
1774
|
+
};
|
|
1775
|
+
}
|
|
1776
|
+
function withBundledPythonEnvironment(env) {
|
|
1777
|
+
return {
|
|
1778
|
+
...env,
|
|
1779
|
+
UV_PROJECT_ENVIRONMENT: BUNDLED_PYTHON_ENVIRONMENT
|
|
1780
|
+
};
|
|
1781
|
+
}
|
|
1782
|
+
function buildFoundationPythonCommand(entrypoint, args = []) {
|
|
1783
|
+
const target = entrypoint === "-c" ? entrypoint : join3(foundationProject, "src", entrypoint);
|
|
1784
|
+
const commandArgs = [
|
|
1785
|
+
"run",
|
|
1786
|
+
"--frozen",
|
|
1787
|
+
"--quiet",
|
|
1788
|
+
"--project",
|
|
1789
|
+
foundationProject,
|
|
1790
|
+
"python",
|
|
1791
|
+
target,
|
|
1792
|
+
...args
|
|
1793
|
+
];
|
|
1794
|
+
return {
|
|
1795
|
+
command: "uv",
|
|
1796
|
+
commandArgs,
|
|
1797
|
+
displayCommand: ["uv", ...commandArgs]
|
|
1798
|
+
};
|
|
1799
|
+
}
|
|
1800
|
+
function withFoundationPythonEnvironment(env) {
|
|
1801
|
+
return {
|
|
1802
|
+
...env,
|
|
1803
|
+
UV_PROJECT_ENVIRONMENT: FOUNDATION_PYTHON_ENVIRONMENT
|
|
1804
|
+
};
|
|
1805
|
+
}
|
|
1806
|
+
var ProcessCancelledError = class extends Error {
|
|
1807
|
+
constructor(message = "Process cancelled.") {
|
|
1808
|
+
super(message);
|
|
1809
|
+
this.name = "ProcessCancelledError";
|
|
1810
|
+
}
|
|
1811
|
+
};
|
|
1812
|
+
async function openProcessLog(path, exclusive, append) {
|
|
1813
|
+
const flags = constants.O_WRONLY | constants.O_CREAT | (exclusive ? constants.O_EXCL : append ? constants.O_APPEND : constants.O_TRUNC) | (process.platform === "win32" ? 0 : constants.O_NOFOLLOW);
|
|
1814
|
+
const handle = await open2(path, flags, 384);
|
|
1815
|
+
try {
|
|
1816
|
+
return handle.createWriteStream({ autoClose: true });
|
|
1817
|
+
} catch (error) {
|
|
1818
|
+
await handle.close();
|
|
1819
|
+
throw error;
|
|
1820
|
+
}
|
|
1821
|
+
}
|
|
1822
|
+
async function runLoggedProcess(args) {
|
|
1823
|
+
if (args.logPath) await mkdir2(dirname3(args.logPath), { recursive: true });
|
|
1824
|
+
const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false, args.appendLog ?? false) : null;
|
|
1825
|
+
let stderr = "";
|
|
1826
|
+
try {
|
|
1827
|
+
const exitCode = await new Promise((resolvePromise, reject) => {
|
|
1828
|
+
const child = spawn(args.command, args.commandArgs, {
|
|
1829
|
+
cwd: args.cwd ? resolve5(args.cwd) : process.cwd(),
|
|
1830
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1831
|
+
env: args.env ?? process.env,
|
|
1832
|
+
detached: process.platform !== "win32"
|
|
1833
|
+
});
|
|
1834
|
+
let timedOut = false;
|
|
1835
|
+
let cancelled = false;
|
|
1836
|
+
let cancellationError;
|
|
1837
|
+
let cancellationCheckRunning = false;
|
|
1838
|
+
let forceKillTimer = null;
|
|
1839
|
+
const killProcessGroup = (signal) => {
|
|
1840
|
+
if (child.pid && process.platform !== "win32") {
|
|
1841
|
+
try {
|
|
1842
|
+
process.kill(-child.pid, signal);
|
|
1843
|
+
return;
|
|
1844
|
+
} catch {
|
|
1845
|
+
}
|
|
1846
|
+
}
|
|
1847
|
+
child.kill(signal);
|
|
1848
|
+
};
|
|
1849
|
+
const requestStop = (signal = "SIGTERM") => {
|
|
1850
|
+
killProcessGroup(signal);
|
|
1851
|
+
if (!forceKillTimer) {
|
|
1852
|
+
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), args.shutdownGraceMs ?? 5e3);
|
|
1853
|
+
forceKillTimer.unref();
|
|
1854
|
+
}
|
|
1855
|
+
};
|
|
1856
|
+
const onSigint = () => {
|
|
1857
|
+
cancelled = true;
|
|
1858
|
+
requestStop("SIGINT");
|
|
1859
|
+
};
|
|
1860
|
+
const onSigterm = () => {
|
|
1861
|
+
cancelled = true;
|
|
1862
|
+
requestStop("SIGTERM");
|
|
1863
|
+
};
|
|
1864
|
+
process.once("SIGINT", onSigint);
|
|
1865
|
+
process.once("SIGTERM", onSigterm);
|
|
1866
|
+
const timer = args.timeoutMs ? setTimeout(() => {
|
|
1867
|
+
timedOut = true;
|
|
1868
|
+
requestStop();
|
|
1869
|
+
}, args.timeoutMs) : null;
|
|
1870
|
+
const cancellationTimer = args.shouldCancel ? setInterval(() => {
|
|
1871
|
+
if (cancellationCheckRunning || cancelled) return;
|
|
1872
|
+
cancellationCheckRunning = true;
|
|
1873
|
+
Promise.resolve(args.shouldCancel?.()).then((requested) => {
|
|
1874
|
+
if (!requested || cancelled) return;
|
|
1875
|
+
cancelled = true;
|
|
1876
|
+
requestStop();
|
|
1877
|
+
}).catch((error) => {
|
|
1878
|
+
cancellationError = error;
|
|
1879
|
+
requestStop();
|
|
1880
|
+
}).finally(() => {
|
|
1881
|
+
cancellationCheckRunning = false;
|
|
1882
|
+
});
|
|
1883
|
+
}, args.cancelPollMs ?? 250) : null;
|
|
1884
|
+
cancellationTimer?.unref();
|
|
1885
|
+
const clearProcessTimers = () => {
|
|
1886
|
+
if (timer) clearTimeout(timer);
|
|
1887
|
+
if (cancellationTimer) clearInterval(cancellationTimer);
|
|
1888
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
1889
|
+
process.off("SIGINT", onSigint);
|
|
1890
|
+
process.off("SIGTERM", onSigterm);
|
|
1891
|
+
};
|
|
1892
|
+
if (logStream) {
|
|
1893
|
+
child.stdout.pipe(logStream, { end: false });
|
|
1894
|
+
child.stderr.pipe(logStream, { end: false });
|
|
1895
|
+
}
|
|
1896
|
+
forwardStreamLines(child.stdout, (line) => {
|
|
1897
|
+
const progress = parseProgressLine(line);
|
|
1898
|
+
if (progress) {
|
|
1899
|
+
reportInBackground(() => args.reporter?.onProgress?.(progress));
|
|
1900
|
+
return;
|
|
1901
|
+
}
|
|
1902
|
+
args.onLine?.(line, "stdout");
|
|
1903
|
+
if (args.reporter?.verbose) {
|
|
1904
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stdout", message: line }));
|
|
1905
|
+
}
|
|
1906
|
+
});
|
|
1907
|
+
forwardStreamLines(child.stderr, (line) => {
|
|
1908
|
+
stderr += `${line}
|
|
1909
|
+
`;
|
|
1910
|
+
args.onLine?.(line, "stderr");
|
|
1911
|
+
if (args.reporter?.verbose) {
|
|
1912
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stderr", message: line }));
|
|
1913
|
+
}
|
|
1914
|
+
});
|
|
1915
|
+
child.on("error", (error) => {
|
|
1916
|
+
clearProcessTimers();
|
|
1917
|
+
reject(timedOut ? new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`) : error);
|
|
1918
|
+
});
|
|
1919
|
+
child.on("close", (code) => {
|
|
1920
|
+
if (timedOut || cancelled || cancellationError || args.terminateProcessGroupOnExit) {
|
|
1921
|
+
killProcessGroup("SIGKILL");
|
|
1922
|
+
}
|
|
1923
|
+
clearProcessTimers();
|
|
1924
|
+
if (timedOut) {
|
|
1925
|
+
reject(new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`));
|
|
1926
|
+
return;
|
|
1927
|
+
}
|
|
1928
|
+
if (cancellationError) {
|
|
1929
|
+
reject(cancellationError);
|
|
1930
|
+
return;
|
|
1931
|
+
}
|
|
1932
|
+
if (cancelled) {
|
|
1933
|
+
reject(new ProcessCancelledError(`${args.stage} was cancelled.`));
|
|
1934
|
+
return;
|
|
1935
|
+
}
|
|
1936
|
+
resolvePromise(code ?? 1);
|
|
1937
|
+
});
|
|
1938
|
+
});
|
|
1939
|
+
return { exitCode, stderr };
|
|
1940
|
+
} finally {
|
|
1941
|
+
if (logStream) {
|
|
1942
|
+
await new Promise((resolveEnd) => {
|
|
1943
|
+
logStream.end(resolveEnd);
|
|
1944
|
+
});
|
|
1945
|
+
}
|
|
1946
|
+
}
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
// src/local-runtime/assets.ts
|
|
1950
|
+
import { createHash as createHash3 } from "crypto";
|
|
1951
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
1952
|
+
import { readFile as readFile2, readdir as readdir3, realpath as realpath2, stat as stat2 } from "fs/promises";
|
|
1953
|
+
import { basename, dirname as dirname5, join as join5, relative as relative2, sep as sep2 } from "path";
|
|
1954
|
+
|
|
1955
|
+
// src/local-runtime/huggingface-cache.ts
|
|
1956
|
+
import { homedir as homedir2 } from "os";
|
|
1957
|
+
import { dirname as dirname4, join as join4, resolve as resolve6 } from "path";
|
|
1958
|
+
function minimalMachineLearningEnvironment(env) {
|
|
1959
|
+
const exact = /* @__PURE__ */ new Set([
|
|
1960
|
+
"PATH",
|
|
1961
|
+
"HOME",
|
|
1962
|
+
"USER",
|
|
1963
|
+
"LOGNAME",
|
|
1964
|
+
"SHELL",
|
|
1965
|
+
"TMPDIR",
|
|
1966
|
+
"TEMP",
|
|
1967
|
+
"TMP",
|
|
1968
|
+
"LANG",
|
|
1969
|
+
"LANGUAGE",
|
|
1970
|
+
"LD_LIBRARY_PATH",
|
|
1971
|
+
"DYLD_LIBRARY_PATH",
|
|
1972
|
+
"XDG_CACHE_HOME",
|
|
1973
|
+
"SSL_CERT_FILE",
|
|
1974
|
+
"REQUESTS_CA_BUNDLE",
|
|
1975
|
+
"CURL_CA_BUNDLE",
|
|
1976
|
+
"NO_PROXY",
|
|
1977
|
+
"no_proxy",
|
|
1978
|
+
"UV_CACHE_DIR",
|
|
1979
|
+
"UV_PYTHON",
|
|
1980
|
+
"HF_HOME",
|
|
1981
|
+
"HF_HUB_CACHE",
|
|
1982
|
+
"HUGGINGFACE_HUB_CACHE",
|
|
1983
|
+
"HF_ENDPOINT",
|
|
1984
|
+
"HF_TOKEN",
|
|
1985
|
+
"HF_HUB_OFFLINE",
|
|
1986
|
+
"HF_HUB_DISABLE_TELEMETRY",
|
|
1987
|
+
"TRANSFORMERS_OFFLINE",
|
|
1988
|
+
"HF_HUB_ENABLE_HF_TRANSFER",
|
|
1989
|
+
"HF_HUB_DOWNLOAD_TIMEOUT",
|
|
1990
|
+
"HF_HUB_ETAG_TIMEOUT",
|
|
1991
|
+
"TT_HF_DOWNLOAD_MAX_ATTEMPTS",
|
|
1992
|
+
"PYTHONUTF8",
|
|
1993
|
+
"PYTHONIOENCODING"
|
|
1994
|
+
]);
|
|
1995
|
+
const prefixes = [
|
|
1996
|
+
"LC_",
|
|
1997
|
+
"CUDA_",
|
|
1998
|
+
"NVIDIA_",
|
|
1999
|
+
"NCCL_",
|
|
2000
|
+
"OMP_",
|
|
2001
|
+
"MKL_",
|
|
2002
|
+
"TORCH_",
|
|
2003
|
+
"PYTORCH_",
|
|
2004
|
+
"TOKENIZERS_",
|
|
2005
|
+
"HF_XET_"
|
|
2006
|
+
];
|
|
2007
|
+
const result = Object.fromEntries(Object.entries(env).filter(
|
|
2008
|
+
([key]) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix))
|
|
2009
|
+
));
|
|
2010
|
+
result.TORCH_DISABLE_NATIVE_JIT ??= "1";
|
|
2011
|
+
return result;
|
|
2012
|
+
}
|
|
2013
|
+
function resolveHuggingFaceCacheLayout(modelCache) {
|
|
2014
|
+
const hfHome = resolve6(modelCache);
|
|
2015
|
+
return {
|
|
2016
|
+
hfHome,
|
|
2017
|
+
hubCache: join4(hfHome, "hub")
|
|
2018
|
+
};
|
|
2019
|
+
}
|
|
2020
|
+
function withHuggingFaceCacheEnvironment(env, modelCache) {
|
|
2021
|
+
const inheritedHub = env.HF_HUB_CACHE ?? env.HUGGINGFACE_HUB_CACHE;
|
|
2022
|
+
const canonicalHome = modelCache ?? env.HF_HOME ?? (inheritedHub ? dirname4(resolve6(inheritedHub)) : join4(homedir2(), ".cache", "huggingface"));
|
|
2023
|
+
const layout = resolveHuggingFaceCacheLayout(canonicalHome);
|
|
2024
|
+
const result = {
|
|
2025
|
+
...env,
|
|
2026
|
+
HF_HOME: layout.hfHome,
|
|
2027
|
+
HF_HUB_CACHE: layout.hubCache,
|
|
2028
|
+
HUGGINGFACE_HUB_CACHE: layout.hubCache
|
|
2029
|
+
};
|
|
2030
|
+
delete result.TRANSFORMERS_CACHE;
|
|
2031
|
+
delete result.PYTORCH_TRANSFORMERS_CACHE;
|
|
2032
|
+
delete result.PYTORCH_PRETRAINED_BERT_CACHE;
|
|
2033
|
+
return result;
|
|
2034
|
+
}
|
|
2035
|
+
function withOfflineHuggingFaceCacheEnvironment(env, modelCache) {
|
|
2036
|
+
return {
|
|
2037
|
+
...withHuggingFaceCacheEnvironment(env, modelCache),
|
|
2038
|
+
HF_HUB_OFFLINE: "1",
|
|
2039
|
+
TRANSFORMERS_OFFLINE: "1"
|
|
2040
|
+
};
|
|
2041
|
+
}
|
|
2042
|
+
|
|
2043
|
+
// src/local-runtime/assets.ts
|
|
2044
|
+
var COMMIT_SHA = /^[0-9a-f]{40}$/;
|
|
2045
|
+
var SpecLockMismatchError = class extends Error {
|
|
2046
|
+
constructor(message) {
|
|
2047
|
+
super(message);
|
|
2048
|
+
this.name = "SpecLockMismatchError";
|
|
2049
|
+
}
|
|
2050
|
+
};
|
|
2051
|
+
function assetsRoot(env = process.env) {
|
|
2052
|
+
return join5(getTunedTensorHome(env), "assets");
|
|
2053
|
+
}
|
|
2054
|
+
function snapshotDirectory(hubCache, kind, repo, revision) {
|
|
2055
|
+
return join5(hubCache, `${kind}s--${repo.replaceAll("/", "--")}`, "snapshots", revision);
|
|
2056
|
+
}
|
|
2057
|
+
async function contentDigest(path, algorithm) {
|
|
2058
|
+
const size = (await stat2(path)).size;
|
|
2059
|
+
const hash = algorithm === "sha256" ? createHash3("sha256") : createHash3("sha1").update(`blob ${size}\0`);
|
|
2060
|
+
for await (const chunk of createReadStream2(path, { highWaterMark: 8 * 1024 * 1024 })) hash.update(chunk);
|
|
2061
|
+
return `${algorithm}:${hash.digest("hex")}`;
|
|
2062
|
+
}
|
|
2063
|
+
async function listFiles(root, directory = root) {
|
|
2064
|
+
const files = [];
|
|
2065
|
+
for (const entry of await readdir3(directory, { withFileTypes: true })) {
|
|
2066
|
+
const child = join5(directory, entry.name);
|
|
2067
|
+
if (entry.isDirectory()) files.push(...await listFiles(root, child));
|
|
2068
|
+
else files.push(relative2(root, child).split(sep2).join("/"));
|
|
2069
|
+
}
|
|
2070
|
+
return files.sort();
|
|
2071
|
+
}
|
|
2072
|
+
async function snapshotDigests(snapshot, paths) {
|
|
2073
|
+
const files = {};
|
|
2074
|
+
let sizeBytes = 0;
|
|
2075
|
+
for (const path of paths ?? await listFiles(snapshot)) {
|
|
2076
|
+
const local = join5(snapshot, ...path.split("/"));
|
|
2077
|
+
const target = await realpath2(local);
|
|
2078
|
+
sizeBytes += (await stat2(target)).size;
|
|
2079
|
+
const blob = target !== local && basename(dirname5(target)) === "blobs" ? basename(target).toLowerCase() : "";
|
|
2080
|
+
files[path] = await contentDigest(target, /^[0-9a-f]{40}$/.test(blob) ? "git-sha1" : "sha256");
|
|
2081
|
+
if (/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(blob) && files[path].split(":")[1] !== blob) {
|
|
2082
|
+
throw new SpecLockMismatchError(`Cached file ${path} failed its checksum. Remove the corrupt blob ${target} and prefetch it again.`);
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
return { files, sizeBytes };
|
|
2086
|
+
}
|
|
2087
|
+
function manifestPath(kind, repo, revision, env) {
|
|
2088
|
+
return join5(assetsRoot(env), `${kind}s`, repo.replaceAll("/", "--"), revision, "manifest.json");
|
|
2089
|
+
}
|
|
2090
|
+
async function recordAssetDownload(args) {
|
|
2091
|
+
const path = manifestPath(args.kind, args.repo, args.revision, args.env);
|
|
2092
|
+
let paths = args.paths;
|
|
2093
|
+
if (paths) {
|
|
2094
|
+
const previous = Object.keys((await readJsonIfPresent(path))?.files ?? {});
|
|
2095
|
+
const stillCached = await Promise.all(previous.map(async (file) => await stat2(join5(args.snapshotPath, ...file.split("/"))).then(() => file, () => null)));
|
|
2096
|
+
paths = [.../* @__PURE__ */ new Set([...stillCached.filter((file) => file !== null), ...paths])].sort();
|
|
2097
|
+
}
|
|
2098
|
+
const { files, sizeBytes } = await snapshotDigests(args.snapshotPath, paths);
|
|
2099
|
+
const manifest = {
|
|
2100
|
+
version: 1,
|
|
2101
|
+
kind: args.kind,
|
|
2102
|
+
repo: args.repo,
|
|
2103
|
+
revision: args.revision,
|
|
2104
|
+
fetched_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2105
|
+
snapshot_path: args.snapshotPath,
|
|
2106
|
+
size_bytes: sizeBytes,
|
|
2107
|
+
files
|
|
2108
|
+
};
|
|
2109
|
+
await writeJsonAtomic(path, manifest);
|
|
2110
|
+
return { manifest, path };
|
|
2111
|
+
}
|
|
2112
|
+
async function readJsonIfPresent(path) {
|
|
2113
|
+
try {
|
|
2114
|
+
return JSON.parse(await readFile2(path, "utf8"));
|
|
2115
|
+
} catch (error) {
|
|
2116
|
+
if (error.code === "ENOENT") return void 0;
|
|
2117
|
+
throw error;
|
|
2118
|
+
}
|
|
2119
|
+
}
|
|
2120
|
+
async function listAssets(env) {
|
|
2121
|
+
const root = assetsRoot(env);
|
|
2122
|
+
const found = [];
|
|
2123
|
+
for (const kind of ["datasets", "models"]) {
|
|
2124
|
+
const repos = await readdir3(join5(root, kind)).catch(() => []);
|
|
2125
|
+
for (const repo of repos) {
|
|
2126
|
+
for (const revision of await readdir3(join5(root, kind, repo)).catch(() => [])) {
|
|
2127
|
+
const path = join5(root, kind, repo, revision, "manifest.json");
|
|
2128
|
+
const manifest = await readJsonIfPresent(path);
|
|
2129
|
+
if (manifest) found.push({ ...manifest, manifest_path: path });
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2133
|
+
return found.sort((a, b) => b.fetched_at.localeCompare(a.fetched_at));
|
|
2134
|
+
}
|
|
2135
|
+
async function findAsset(reference, env) {
|
|
2136
|
+
const [repo, revision] = reference.split("@");
|
|
2137
|
+
const match = (await listAssets(env)).find((asset) => asset.repo === repo && (!revision || asset.revision.startsWith(revision.toLowerCase())));
|
|
2138
|
+
if (!match) throw new Error(`No downloaded asset matches ${reference}. Run \`tt assets list\` to see downloads.`);
|
|
2139
|
+
return match;
|
|
2140
|
+
}
|
|
2141
|
+
async function verifyAsset(manifest) {
|
|
2142
|
+
const problems = [];
|
|
2143
|
+
for (const [path, expected] of Object.entries(manifest.files)) {
|
|
2144
|
+
const local = join5(manifest.snapshot_path, ...path.split("/"));
|
|
2145
|
+
const target = await realpath2(local).catch(() => null);
|
|
2146
|
+
if (!target) {
|
|
2147
|
+
problems.push(`${path}: missing from ${manifest.snapshot_path}`);
|
|
2148
|
+
continue;
|
|
2149
|
+
}
|
|
2150
|
+
const algorithm = expected.startsWith("git-sha1:") ? "git-sha1" : "sha256";
|
|
2151
|
+
if (await contentDigest(target, algorithm) !== expected) problems.push(`${path}: content does not match ${expected}`);
|
|
2152
|
+
}
|
|
2153
|
+
return { ok: problems.length === 0, problems };
|
|
2154
|
+
}
|
|
2155
|
+
function specLockPath(specPath) {
|
|
2156
|
+
const name = basename(specPath).replace(/^\.(.+)\.tt-(?:local|agent)-[^/\\]*\.json$/, "$1");
|
|
2157
|
+
return join5(dirname5(specPath), `${name.replace(/\.json$/i, "")}.lock.json`);
|
|
2158
|
+
}
|
|
2159
|
+
async function readSpecLock(specPath) {
|
|
2160
|
+
const path = specLockPath(specPath);
|
|
2161
|
+
const lock = await readJsonIfPresent(path);
|
|
2162
|
+
if (lock && lock.version !== 1) throw new Error(`${path} has unsupported version ${String(lock.version)}.`);
|
|
2163
|
+
return lock;
|
|
2164
|
+
}
|
|
2165
|
+
async function writeSpecLockEntry(specPath, key, entry) {
|
|
2166
|
+
const lock = await readSpecLock(specPath) ?? { version: 1 };
|
|
2167
|
+
await writeJsonAtomic(specLockPath(specPath), { ...lock, version: 1, [key]: sortedEntry(entry) });
|
|
2168
|
+
}
|
|
2169
|
+
function sortedEntry(entry) {
|
|
2170
|
+
return {
|
|
2171
|
+
repo: entry.repo,
|
|
2172
|
+
revision: entry.revision,
|
|
2173
|
+
files: Object.fromEntries(Object.entries(entry.files).sort(([a], [b]) => a.localeCompare(b)))
|
|
2174
|
+
};
|
|
2175
|
+
}
|
|
2176
|
+
var PREFETCH_COMMAND = {
|
|
2177
|
+
dataset: "tt datasets prefetch",
|
|
2178
|
+
base_model: "tt models prefetch"
|
|
2179
|
+
};
|
|
2180
|
+
async function checkSpecLock(specPath, key, actual) {
|
|
2181
|
+
const locked = (await readSpecLock(specPath))?.[key];
|
|
2182
|
+
if (!locked) {
|
|
2183
|
+
await writeSpecLockEntry(specPath, key, actual);
|
|
2184
|
+
return "recorded";
|
|
2185
|
+
}
|
|
2186
|
+
const lockFile = basename(specLockPath(specPath));
|
|
2187
|
+
const update = `If this change is intended, run \`${PREFETCH_COMMAND[key]}\` with this spec to update ${lockFile}.`;
|
|
2188
|
+
const label = key === "dataset" ? "dataset" : "base model";
|
|
2189
|
+
if (locked.repo !== actual.repo || locked.revision !== actual.revision) {
|
|
2190
|
+
throw new SpecLockMismatchError(
|
|
2191
|
+
`The spec's ${label} is ${actual.repo}@${actual.revision}, but ${lockFile} pins ${locked.repo}@${locked.revision}. ${update}`
|
|
2192
|
+
);
|
|
2193
|
+
}
|
|
2194
|
+
if (key === "base_model") {
|
|
2195
|
+
for (const path of Object.keys(locked.files)) {
|
|
2196
|
+
if (actual.files[path] === void 0) {
|
|
2197
|
+
throw new SpecLockMismatchError(`base model file ${path} is missing from the cached snapshot. ${update}`);
|
|
2198
|
+
}
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
for (const [path, digest] of Object.entries(actual.files)) {
|
|
2202
|
+
if (locked.files[path] === void 0) {
|
|
2203
|
+
throw new SpecLockMismatchError(`${label} file ${path} is not in ${lockFile}. ${update}`);
|
|
2204
|
+
}
|
|
2205
|
+
if (locked.files[path] !== digest) {
|
|
2206
|
+
throw new SpecLockMismatchError(
|
|
2207
|
+
`${label} file ${path} is ${digest}, but ${lockFile} expects ${locked.files[path]}. The cached file differs from the one this project was locked to. ${update}`
|
|
2208
|
+
);
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
return "matched";
|
|
2212
|
+
}
|
|
2213
|
+
function lockedRevision(lock, key, repo) {
|
|
2214
|
+
const entry = lock?.[key];
|
|
2215
|
+
return entry?.repo === repo && COMMIT_SHA.test(entry.revision) ? entry.revision : void 0;
|
|
2216
|
+
}
|
|
2217
|
+
function applySpecLock(request, lock) {
|
|
2218
|
+
const source = request.dataset_prebuilt?.huggingface;
|
|
2219
|
+
if (!source || source.revision) return request;
|
|
2220
|
+
const revision = lockedRevision(lock, "dataset", source.repo);
|
|
2221
|
+
if (!revision) return request;
|
|
2222
|
+
return { ...request, dataset_prebuilt: { ...request.dataset_prebuilt, huggingface: { ...source, revision } } };
|
|
2223
|
+
}
|
|
2224
|
+
async function enforceSpecLock(args) {
|
|
2225
|
+
if (args.config.dryRun) return;
|
|
2226
|
+
const dataset = args.request.dataset_source;
|
|
2227
|
+
if (dataset) {
|
|
2228
|
+
const paths = Object.values(dataset.files).flatMap((file) => file ? [file.path] : []);
|
|
2229
|
+
const { files: files2 } = await snapshotDigests(dataset.snapshot_path, paths);
|
|
2230
|
+
await checkSpecLock(args.specPath, "dataset", { repo: dataset.repo, revision: dataset.revision, files: files2 });
|
|
2231
|
+
}
|
|
2232
|
+
if (args.config.paths.baseModel || args.config.gpu) return;
|
|
2233
|
+
const model = resolveTrainingModel(args.request.spec_snapshot.base_model);
|
|
2234
|
+
const revision = resolveRequestedBaseModelRevision(model.id, args.request.hyperparameters.base_model_revision);
|
|
2235
|
+
if (!revision) return;
|
|
2236
|
+
const hubCache = withHuggingFaceCacheEnvironment(process.env, args.config.paths.modelCache).HF_HUB_CACHE;
|
|
2237
|
+
const snapshot = snapshotDirectory(hubCache, "model", model.id, revision);
|
|
2238
|
+
if (!await stat2(snapshot).then(() => true, () => false)) {
|
|
2239
|
+
if ((await readSpecLock(args.specPath))?.base_model) {
|
|
2240
|
+
throw new SpecLockMismatchError(
|
|
2241
|
+
`Base model ${model.id}@${revision} is not downloaded (${snapshot}). Run \`tt models prefetch\` with this spec first.`
|
|
2242
|
+
);
|
|
2243
|
+
}
|
|
2244
|
+
const { prefetchBaseModel: prefetchBaseModel2 } = await import("./prefetch-GP3OPUUL.js");
|
|
2245
|
+
await prefetchBaseModel2({ request: args.request, config: args.config, reporter: args.reporter });
|
|
2246
|
+
}
|
|
2247
|
+
const { files } = await snapshotDigests(snapshot);
|
|
2248
|
+
await checkSpecLock(args.specPath, "base_model", { repo: model.id, revision, files });
|
|
2249
|
+
}
|
|
2250
|
+
|
|
2251
|
+
// src/local-runtime/prefetch.ts
|
|
2252
|
+
var NEMOTRON_BF16_FILE_DIGESTS = {
|
|
2253
|
+
".gitattributes": { algorithm: "sha1", digest: "fe3f694ce14b577c9833fca6e83c5e26a7119127" },
|
|
2254
|
+
"LICENSE": { algorithm: "sha1", digest: "4d76cc87c9edb280753e3bee3b499ae461113835" },
|
|
2255
|
+
"README.md": { algorithm: "sha1", digest: "8ec43ee9f3ee345202e4a6ef752c51b2619f7425" },
|
|
2256
|
+
"accuracy_plot.png": { algorithm: "sha256", digest: "1397995f8a5d34d819a069d86c02914990e9ecfc2fce101e233924da9cf5ddcf" },
|
|
2257
|
+
"agentic_coding_benchmarks.png": { algorithm: "sha256", digest: "06cf5486ae94ecd7cc6f91ae3b4bae21c7435253a294cb89f54f7ec42eda8847" },
|
|
2258
|
+
"bias.md": { algorithm: "sha1", digest: "cdcc8055f965222fabddabcabf22885d3a8bacc1" },
|
|
2259
|
+
"chat_template.jinja": { algorithm: "sha1", digest: "d85b0c772f8fe585063847c5f6bf5ec48eb210be" },
|
|
2260
|
+
"config.json": { algorithm: "sha1", digest: "993b612989e6aa67da45f908ea5983cb777e7678" },
|
|
2261
|
+
"explainability.md": { algorithm: "sha1", digest: "2435f23c865e7daba6a6eceef2f214b86df75d24" },
|
|
2262
|
+
"generation_config.json": { algorithm: "sha1", digest: "a41201df53f9d0769947989353b5d02e70491c56" },
|
|
2263
|
+
"model.safetensors.index.json": { algorithm: "sha1", digest: "3cd409443b1896d9adc5c3981af301841f85329a" },
|
|
2264
|
+
"special_tokens_map.json": { algorithm: "sha1", digest: "0451f37912edb7f8a4bf04d77eef13f6130f0515" },
|
|
2265
|
+
"tokenizer.json": { algorithm: "sha256", digest: "623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7" },
|
|
2266
|
+
"tokenizer_config.json": { algorithm: "sha1", digest: "c96e5ad00986bfa0b666d4bc425666f1c51d6d1b" },
|
|
2267
|
+
"model-00001-of-00014.safetensors": { algorithm: "sha256", digest: "7a3e74d24969eac6657cb9b6422091fd062b7d30fd4bc90ecee6c3d7f692626a" },
|
|
2268
|
+
"model-00002-of-00014.safetensors": { algorithm: "sha256", digest: "a80c3846e959f6dda34aa031cbb668b766643691ac7e5c9f43f62e6fb1221059" },
|
|
2269
|
+
"model-00003-of-00014.safetensors": { algorithm: "sha256", digest: "996bf3a46668e14075cb82259390267c2b1f71446caeb3a5f1f9e0bd7a079ae7" },
|
|
2270
|
+
"model-00004-of-00014.safetensors": { algorithm: "sha256", digest: "cce1d79e7ff2e2b746b9d97166156aa1dcec796116675cc7fd1aa70a8d73ccb2" },
|
|
2271
|
+
"model-00005-of-00014.safetensors": { algorithm: "sha256", digest: "9338f59d3fd5f074288c71a9559452d5b5c581da00e9ccb9fe7aa2ce9b04d2ad" },
|
|
2272
|
+
"model-00006-of-00014.safetensors": { algorithm: "sha256", digest: "e947fa4097c68263ba3d097fceb32a300a22cebd9d909fb5198af7d18a4022fe" },
|
|
2273
|
+
"model-00007-of-00014.safetensors": { algorithm: "sha256", digest: "c7362630bf4256275ca58354ed8f31506cec67ffab30573e753703fd5ae48795" },
|
|
2274
|
+
"model-00008-of-00014.safetensors": { algorithm: "sha256", digest: "1b33a5a6b5505b25b57c90ddda19e9cd1598176468917df0c56116fa6f6072cb" },
|
|
2275
|
+
"model-00009-of-00014.safetensors": { algorithm: "sha256", digest: "fcc9b48caff427beaf1f87b78cf4ab42bc2a3c43b3c487bccb013ba22d7468b2" },
|
|
2276
|
+
"model-00010-of-00014.safetensors": { algorithm: "sha256", digest: "d1924fd54d1b77a15aff8df591a317e8a82a58e46d6f5742fa21e90e0be3cfa6" },
|
|
2277
|
+
"model-00011-of-00014.safetensors": { algorithm: "sha256", digest: "e9d0b83bcf04bd4d5758b94e6495a3f1595d121d82faffaa27ce6a884e2c8757" },
|
|
2278
|
+
"model-00012-of-00014.safetensors": { algorithm: "sha256", digest: "fce184f112cb0b97024cadc60f993f6f660e365d04e21c240fb3c75747bd3a01" },
|
|
2279
|
+
"model-00013-of-00014.safetensors": { algorithm: "sha256", digest: "f427d037d2a3e4d44323325aa38571e3551e8ad702f469c3dacc4e59ea2e50d1" },
|
|
2280
|
+
"model-00014-of-00014.safetensors": { algorithm: "sha256", digest: "64577b275ca4e7e5266eae0903674f7f46ec2a8cbf4f4f1a3207f80d503cd1d0" },
|
|
2281
|
+
"privacy.md": { algorithm: "sha1", digest: "e3bf30aa1c42a488298ef37383647dc99e1cd39e" },
|
|
2282
|
+
"safety.md": { algorithm: "sha1", digest: "f53bf94a4635862eed5cf3e0bc799b93f81d20ac" }
|
|
2283
|
+
};
|
|
2284
|
+
var MUSE_GLIMMER_FILE_DIGESTS = {
|
|
2285
|
+
".gitattributes": { algorithm: "sha1", digest: "52373fe24473b1aa44333d318f578ae6bf04b49b" },
|
|
2286
|
+
"LICENSE": { algorithm: "sha1", digest: "d645695673349e3947e8e5ae42332d0ac3164cd7" },
|
|
2287
|
+
"README.md": { algorithm: "sha1", digest: "a25f87c1c1fbd88f367a552fd14427baa8cb947e" },
|
|
2288
|
+
"USAGE_POLICY.md": { algorithm: "sha1", digest: "1a9ed6cffc54585daf9cc904c07cb6e436a9444b" },
|
|
2289
|
+
"chat_template.jinja": { algorithm: "sha1", digest: "7507f3c9f38809152732c045df3977848f3916a6" },
|
|
2290
|
+
"config.json": { algorithm: "sha1", digest: "190826dc834c13c86b8dc68e775a888a852d7c34" },
|
|
2291
|
+
"generation_config.json": { algorithm: "sha1", digest: "b69a50a4239f42707d54a479e0143b8ee56c8bcc" },
|
|
2292
|
+
"model.safetensors.index.json": { algorithm: "sha1", digest: "00b257ef0c8bce1b001a6266f6b1f66f4fe16a6b" },
|
|
2293
|
+
"processor_config.json": { algorithm: "sha1", digest: "ec9a07be13ea1dadf7395131871753b13f406c26" },
|
|
2294
|
+
"tokenizer_config.json": { algorithm: "sha1", digest: "fe7f0bb90fb0d0288e3a505974ae0e4baf1d3e61" },
|
|
2295
|
+
"tokenizer.json": { algorithm: "sha256", digest: "c9dbee66967b58f31a7c27f723c3760da3526ccd0427578e8905b0abb0031c4d" },
|
|
2296
|
+
"model-00001-of-00002.safetensors": { algorithm: "sha256", digest: "8eef61530e1283642c77ce2e6721feb5c6f348fa055c00e90f2844a136372694" },
|
|
2297
|
+
"model-00002-of-00002.safetensors": { algorithm: "sha256", digest: "b58cc2144ba1ba1af4420f67f4ca3ced7f09298510b80464cc75018a0be14381" }
|
|
2298
|
+
};
|
|
2299
|
+
function certifiedFileDigests(modelId) {
|
|
2300
|
+
switch (modelId) {
|
|
2301
|
+
case "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16":
|
|
2302
|
+
return NEMOTRON_BF16_FILE_DIGESTS;
|
|
2303
|
+
case "meta-models/Muse-Glimmer-30B":
|
|
2304
|
+
return MUSE_GLIMMER_FILE_DIGESTS;
|
|
2305
|
+
default:
|
|
2306
|
+
return void 0;
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
async function certifiedFileDigest(path, expected) {
|
|
2310
|
+
const metadata = await stat3(path);
|
|
2311
|
+
const digest = createHash4(expected.algorithm);
|
|
2312
|
+
if (expected.algorithm === "sha1") digest.update(`blob ${metadata.size}\0`);
|
|
2313
|
+
for await (const chunk of createReadStream3(path, { highWaterMark: 8 * 1024 * 1024 })) digest.update(chunk);
|
|
2314
|
+
return digest.digest("hex");
|
|
2315
|
+
}
|
|
2316
|
+
function isStrictPhysicalDescendant(parent, child) {
|
|
2317
|
+
const candidate = relative3(parent, child);
|
|
2318
|
+
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep3}`) && !isAbsolute3(candidate);
|
|
2319
|
+
}
|
|
2320
|
+
async function verifySnapshotCacheLocation(path, modelId, revision, modelCache) {
|
|
2321
|
+
const repository = `models--${modelId.replaceAll("/", "--")}`;
|
|
2322
|
+
const layout = resolveHuggingFaceCacheLayout(modelCache);
|
|
2323
|
+
const repositoryRoot = join6(layout.hubCache, repository);
|
|
2324
|
+
const expectedPath = join6(repositoryRoot, "snapshots", revision);
|
|
2325
|
+
const [
|
|
2326
|
+
physicalCacheRoot,
|
|
2327
|
+
physicalHubRoot,
|
|
2328
|
+
physicalRepositoryRoot,
|
|
2329
|
+
physicalBlobRoot,
|
|
2330
|
+
physicalPath,
|
|
2331
|
+
physicalExpectedPath
|
|
2332
|
+
] = await Promise.all([
|
|
2333
|
+
realpath3(layout.hfHome).catch(() => null),
|
|
2334
|
+
realpath3(layout.hubCache).catch(() => null),
|
|
2335
|
+
realpath3(repositoryRoot).catch(() => null),
|
|
2336
|
+
realpath3(join6(repositoryRoot, "blobs")).catch(() => null),
|
|
2337
|
+
realpath3(path).catch(() => null),
|
|
2338
|
+
realpath3(expectedPath).catch(() => null)
|
|
2339
|
+
]);
|
|
2340
|
+
if (!physicalCacheRoot || !physicalHubRoot || !physicalRepositoryRoot || !physicalBlobRoot || !physicalPath || !physicalExpectedPath || physicalPath !== physicalExpectedPath) {
|
|
2341
|
+
throw new Error(
|
|
2342
|
+
`${modelId} must use the certified Hugging Face cache snapshot ${expectedPath}; got ${path}.`
|
|
2343
|
+
);
|
|
2344
|
+
}
|
|
2345
|
+
if (!isStrictPhysicalDescendant(physicalCacheRoot, physicalHubRoot)) {
|
|
2346
|
+
throw new Error(`Hugging Face hub cache escapes its configured cache root: ${layout.hubCache}`);
|
|
2347
|
+
}
|
|
2348
|
+
if (!isStrictPhysicalDescendant(physicalHubRoot, physicalRepositoryRoot)) {
|
|
2349
|
+
throw new Error(`Hugging Face repository escapes its configured hub cache: ${repositoryRoot}`);
|
|
2350
|
+
}
|
|
2351
|
+
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalBlobRoot)) {
|
|
2352
|
+
throw new Error(`Hugging Face blob store escapes its repository: ${join6(repositoryRoot, "blobs")}`);
|
|
2353
|
+
}
|
|
2354
|
+
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalExpectedPath)) {
|
|
2355
|
+
throw new Error(`Hugging Face snapshot escapes its configured repository: ${expectedPath}`);
|
|
2356
|
+
}
|
|
2357
|
+
return { repositoryRoot, physicalBlobRoot };
|
|
2358
|
+
}
|
|
2359
|
+
async function requireReportedCachePath(label, reported, expected) {
|
|
2360
|
+
const [physicalReported, physicalExpected] = await Promise.all([
|
|
2361
|
+
reported ? realpath3(reported).catch(() => null) : null,
|
|
2362
|
+
realpath3(expected).catch(() => null)
|
|
2363
|
+
]);
|
|
2364
|
+
if (!physicalReported || !physicalExpected || physicalReported !== physicalExpected) {
|
|
2365
|
+
throw new Error(`Model prefetch returned an unexpected ${label}: ${String(reported)}.`);
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
async function verifyModelPrefetchCacheReport(args) {
|
|
2369
|
+
if (!args.output.snapshot_path) {
|
|
2370
|
+
throw new Error("Model prefetch did not return a snapshot path.");
|
|
2371
|
+
}
|
|
2372
|
+
const effectiveHubCache = resolveHuggingFaceCacheLayout(args.modelCache).hubCache;
|
|
2373
|
+
await Promise.all([
|
|
2374
|
+
requireReportedCachePath("model cache", args.output.model_cache, args.modelCache),
|
|
2375
|
+
requireReportedCachePath("HF home", args.output.hf_home, args.modelCache),
|
|
2376
|
+
requireReportedCachePath("hub cache", args.output.hub_cache, effectiveHubCache)
|
|
2377
|
+
]);
|
|
2378
|
+
await verifySnapshotCacheLocation(
|
|
2379
|
+
args.output.snapshot_path,
|
|
2380
|
+
args.modelId,
|
|
2381
|
+
args.revision,
|
|
2382
|
+
args.modelCache
|
|
2383
|
+
);
|
|
2384
|
+
}
|
|
2385
|
+
async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelCache) {
|
|
2386
|
+
const revision = defaultBaseModelRevision(modelId);
|
|
2387
|
+
if (!revision) return;
|
|
2388
|
+
const digests = certifiedFileDigests(modelId);
|
|
2389
|
+
if (!digests) return;
|
|
2390
|
+
if (!modelCache) {
|
|
2391
|
+
throw new Error(`A certified local snapshot for ${modelId} requires paths.modelCache.`);
|
|
2392
|
+
}
|
|
2393
|
+
const { physicalBlobRoot } = await verifySnapshotCacheLocation(
|
|
2394
|
+
path,
|
|
2395
|
+
modelId,
|
|
2396
|
+
revision,
|
|
2397
|
+
modelCache
|
|
2398
|
+
);
|
|
2399
|
+
const physicalFiles = /* @__PURE__ */ new Map();
|
|
2400
|
+
for (const file of snapshotFiles) {
|
|
2401
|
+
const metadata = await lstat3(file);
|
|
2402
|
+
const target = await realpath3(file);
|
|
2403
|
+
if (metadata.isSymbolicLink()) {
|
|
2404
|
+
if (!isStrictPhysicalDescendant(physicalBlobRoot, target)) {
|
|
2405
|
+
throw new Error(`Certified snapshot symbolic link escapes its Hugging Face blob store: ${file}`);
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
physicalFiles.set(file, target);
|
|
2409
|
+
}
|
|
2410
|
+
const actualFiles = new Set(snapshotFiles.map((file) => relative3(path, file).split("\\").join("/")));
|
|
2411
|
+
const expectedFiles = new Set(Object.keys(digests));
|
|
2412
|
+
const unexpected = [...actualFiles].filter((name) => !expectedFiles.has(name)).sort();
|
|
2413
|
+
if (unexpected.length > 0) {
|
|
2414
|
+
throw new Error(`Certified snapshot contains unexpected file: ${join6(path, unexpected[0])}`);
|
|
2415
|
+
}
|
|
2416
|
+
for (const [name, expected] of Object.entries(digests)) {
|
|
2417
|
+
if (!actualFiles.has(name)) {
|
|
2418
|
+
throw new Error(`Certified snapshot file is missing: ${join6(path, name)}`);
|
|
2419
|
+
}
|
|
2420
|
+
const file = join6(path, name);
|
|
2421
|
+
const physicalFile = physicalFiles.get(file);
|
|
2422
|
+
if (await certifiedFileDigest(physicalFile, expected) !== expected.digest) {
|
|
2423
|
+
throw new Error(`Certified snapshot file checksum mismatch: ${file}`);
|
|
2424
|
+
}
|
|
2425
|
+
if (await realpath3(file) !== physicalFile) {
|
|
2426
|
+
throw new Error(`Certified snapshot file changed during verification: ${file}`);
|
|
2427
|
+
}
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
function buildModelPrefetchPayload(request, config) {
|
|
2431
|
+
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
2432
|
+
const revision = resolveRequestedBaseModelRevision(
|
|
2433
|
+
model.id,
|
|
2434
|
+
request.hyperparameters.base_model_revision
|
|
2435
|
+
);
|
|
2436
|
+
return {
|
|
2437
|
+
base_model: model.id,
|
|
2438
|
+
...revision ? { revision } : {},
|
|
2439
|
+
...config.paths.modelCache ? { model_cache: resolve7(config.paths.modelCache) } : {}
|
|
2440
|
+
};
|
|
2441
|
+
}
|
|
2442
|
+
function buildModelSourcePrefetchPayload(source, config) {
|
|
2443
|
+
const model = resolveTrainingModel(source.repo);
|
|
2444
|
+
const revision = source.revision || isCertifiedTrainingModel(model.id) ? resolveRequestedBaseModelRevision(model.id, source.revision) : void 0;
|
|
2445
|
+
return {
|
|
2446
|
+
base_model: model.id,
|
|
2447
|
+
...revision ? { revision } : { resolve_latest: true },
|
|
2448
|
+
...config.paths.modelCache ? { model_cache: resolve7(config.paths.modelCache) } : {}
|
|
2449
|
+
};
|
|
2450
|
+
}
|
|
2451
|
+
function safeName(value) {
|
|
2452
|
+
return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "model";
|
|
2453
|
+
}
|
|
2454
|
+
function transformersWeightName(name) {
|
|
2455
|
+
const lower = name.split(/[\\/]/).at(-1)?.toLowerCase() ?? "";
|
|
2456
|
+
return lower.endsWith(".safetensors") && !lower.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(lower);
|
|
2457
|
+
}
|
|
2458
|
+
async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
2459
|
+
const root = await lstat3(path).catch(() => null);
|
|
2460
|
+
if (!root) throw new Error(`paths.baseModel is set to ${path}, but that path does not exist.`);
|
|
2461
|
+
if (root.isSymbolicLink()) {
|
|
2462
|
+
throw new Error(`paths.baseModel must not itself be a symbolic link: ${path}`);
|
|
2463
|
+
}
|
|
2464
|
+
if (!root.isDirectory()) {
|
|
2465
|
+
throw new Error(
|
|
2466
|
+
`paths.baseModel must be a Hugging Face snapshot directory, not a standalone file or archive: ${path}`
|
|
2467
|
+
);
|
|
2468
|
+
}
|
|
2469
|
+
const files = [];
|
|
2470
|
+
const visit = async (directory) => {
|
|
2471
|
+
for (const entry of await readdir4(directory, { withFileTypes: true })) {
|
|
2472
|
+
const child = join6(directory, entry.name);
|
|
2473
|
+
if (entry.isSymbolicLink()) {
|
|
2474
|
+
const metadata2 = await stat3(child);
|
|
2475
|
+
if (!metadata2.isFile()) {
|
|
2476
|
+
throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
|
|
2477
|
+
}
|
|
2478
|
+
files.push({ path: child, size: metadata2.size });
|
|
2479
|
+
continue;
|
|
2480
|
+
}
|
|
2481
|
+
if (entry.isDirectory()) {
|
|
2482
|
+
await visit(child);
|
|
2483
|
+
continue;
|
|
2484
|
+
}
|
|
2485
|
+
const metadata = await stat3(child);
|
|
2486
|
+
if (metadata.isFile()) files.push({ path: child, size: metadata.size });
|
|
2487
|
+
}
|
|
2488
|
+
};
|
|
2489
|
+
await visit(path);
|
|
2490
|
+
const nonEmpty = files.filter((file) => file.size > 0);
|
|
2491
|
+
const weights = nonEmpty.filter((file) => transformersWeightName(file.path));
|
|
2492
|
+
if (weights.length === 0) {
|
|
2493
|
+
throw new Error(`Local base-model directory contains no non-empty Transformers model weights: ${path}`);
|
|
2494
|
+
}
|
|
2495
|
+
const required = async (name) => {
|
|
2496
|
+
const metadata = await stat3(join6(path, name)).catch(() => null);
|
|
2497
|
+
return Boolean(metadata?.isFile() && metadata.size > 0);
|
|
2498
|
+
};
|
|
2499
|
+
if (!await required("config.json")) throw new Error(`Local base-model directory is missing config.json: ${path}`);
|
|
2500
|
+
try {
|
|
2501
|
+
const config = JSON.parse(await readFile3(join6(path, "config.json"), "utf8"));
|
|
2502
|
+
assertCertifiedBaseModelConfig(
|
|
2503
|
+
config,
|
|
2504
|
+
`Local base-model config ${join6(path, "config.json")}`,
|
|
2505
|
+
expectedModelId
|
|
2506
|
+
);
|
|
2507
|
+
} catch (error) {
|
|
2508
|
+
const detail = error instanceof Error ? `: ${error.message}` : "";
|
|
2509
|
+
throw new Error(
|
|
2510
|
+
`Local base-model directory has an invalid or unsupported config.json: ${path}${detail}`,
|
|
2511
|
+
{ cause: error }
|
|
2512
|
+
);
|
|
2513
|
+
}
|
|
2514
|
+
if (expectedModelId) {
|
|
2515
|
+
await verifyCertifiedLocalSnapshot(
|
|
2516
|
+
path,
|
|
2517
|
+
expectedModelId,
|
|
2518
|
+
files.map((file) => file.path),
|
|
2519
|
+
modelCache
|
|
2520
|
+
);
|
|
2521
|
+
}
|
|
2522
|
+
const vocabNames = [
|
|
2523
|
+
"tokenizer.json",
|
|
2524
|
+
"tokenizer.model",
|
|
2525
|
+
"sentencepiece.bpe.model",
|
|
2526
|
+
"spiece.model",
|
|
2527
|
+
"vocab.json",
|
|
2528
|
+
"tokenizer.tiktoken"
|
|
2529
|
+
];
|
|
2530
|
+
if (!await required("tokenizer_config.json") || !(await Promise.all(vocabNames.map(required))).some(Boolean)) {
|
|
2531
|
+
throw new Error(`Local base-model directory is missing tokenizer metadata or vocabulary: ${path}`);
|
|
2532
|
+
}
|
|
2533
|
+
const inventory = new Set(nonEmpty.map((file) => resolve7(file.path)));
|
|
2534
|
+
for (const file of files.filter((candidate) => candidate.path.endsWith(".index.json"))) {
|
|
2535
|
+
const parsed = JSON.parse(await readFile3(file.path, "utf8"));
|
|
2536
|
+
if (!parsed.weight_map || typeof parsed.weight_map !== "object" || Array.isArray(parsed.weight_map)) {
|
|
2537
|
+
throw new Error(`Local base-model weight index is invalid: ${file.path}`);
|
|
2538
|
+
}
|
|
2539
|
+
for (const shard of new Set(Object.values(parsed.weight_map))) {
|
|
2540
|
+
if (typeof shard !== "string" || !shard || /^(?:[A-Za-z]:|[/\\])/.test(shard) || shard.split(/[/\\]/).includes("..")) {
|
|
2541
|
+
throw new Error("Local base-model weight index has an invalid indexed weight shard path");
|
|
2542
|
+
}
|
|
2543
|
+
if (!inventory.has(resolve7(path, shard)) || !await required(shard)) {
|
|
2544
|
+
throw new Error(`Local base-model directory is missing indexed weight shard: ${String(shard)}`);
|
|
2545
|
+
}
|
|
2546
|
+
}
|
|
2547
|
+
}
|
|
2548
|
+
return {
|
|
2549
|
+
fileCount: files.length,
|
|
2550
|
+
sizeBytes: files.reduce((total, file) => total + file.size, 0)
|
|
2551
|
+
};
|
|
2552
|
+
}
|
|
2553
|
+
async function prefetchBaseModel(args) {
|
|
2554
|
+
if (!args.request === !args.source) throw new Error("Model prefetch needs either a spec request or a source repo.");
|
|
2555
|
+
const payload = {
|
|
2556
|
+
...args.source ? buildModelSourcePrefetchPayload(args.source, args.config) : buildModelPrefetchPayload(args.request, args.config),
|
|
2557
|
+
...args.localOnly ? { local_files_only: true } : {}
|
|
2558
|
+
};
|
|
2559
|
+
const artifactDir = resolve7(
|
|
2560
|
+
args.config.artifactRoot,
|
|
2561
|
+
args.localOnly ? "verify-base" : "prefetch",
|
|
2562
|
+
`${safeName(payload.base_model)}-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
|
|
2563
|
+
);
|
|
2564
|
+
await mkdir3(artifactDir, { recursive: true });
|
|
2565
|
+
if (args.request && args.config.paths.baseModel) {
|
|
2566
|
+
const localPath = resolve7(args.config.paths.baseModel);
|
|
2567
|
+
const modelId = args.request.spec_snapshot.base_model;
|
|
2568
|
+
const verified = await withProgressHeartbeat(args.reporter, {
|
|
2569
|
+
stage: "model_prefetch",
|
|
2570
|
+
status: "running",
|
|
2571
|
+
message: "Verifying local model checksums. Large cached models can take several minutes."
|
|
2572
|
+
}, () => verifyLocalBaseModel(
|
|
2573
|
+
localPath,
|
|
2574
|
+
modelId,
|
|
2575
|
+
args.config.paths.modelCache
|
|
2576
|
+
));
|
|
2577
|
+
return {
|
|
2578
|
+
ok: true,
|
|
2579
|
+
status: args.localOnly ? "completed" : "skipped",
|
|
2580
|
+
base_model: args.request.spec_snapshot.base_model,
|
|
2581
|
+
local_base_model_path: localPath,
|
|
2582
|
+
file_count: verified.fileCount,
|
|
2583
|
+
size_bytes: verified.sizeBytes,
|
|
2584
|
+
artifact_dir: artifactDir,
|
|
2585
|
+
reason: args.localOnly ? "Verified the configured local base-model artifact; no network access was used." : "paths.baseModel points at a verified local base-model artifact; no Hugging Face download is needed."
|
|
2586
|
+
};
|
|
2587
|
+
}
|
|
2588
|
+
const inputPath = join6(artifactDir, "prefetch-input.json");
|
|
2589
|
+
const outputPath = join6(artifactDir, "prefetch-output.json");
|
|
2590
|
+
const logPath = join6(artifactDir, "prefetch.log");
|
|
2591
|
+
await writeJson(inputPath, payload);
|
|
2592
|
+
const entrypoint = buildBundledPythonCommand(
|
|
2593
|
+
"prefetch.py",
|
|
2594
|
+
["--input", inputPath, "--output", outputPath]
|
|
2595
|
+
);
|
|
2596
|
+
const downloadEvent = {
|
|
2597
|
+
stage: "model_prefetch",
|
|
2598
|
+
status: "running",
|
|
2599
|
+
message: args.localOnly ? "Verifying cached model files. Large models can take several minutes." : "Downloading and verifying model files. Cached files still need checksum verification; large models can take several minutes.",
|
|
2600
|
+
details: {
|
|
2601
|
+
base_model: payload.base_model,
|
|
2602
|
+
model_cache: payload.model_cache ?? null,
|
|
2603
|
+
command: entrypoint.displayCommand,
|
|
2604
|
+
log_path: logPath
|
|
2605
|
+
}
|
|
2606
|
+
};
|
|
2607
|
+
const cacheEnvironment = withHuggingFaceCacheEnvironment(
|
|
2608
|
+
minimalMachineLearningEnvironment(process.env),
|
|
2609
|
+
payload.model_cache
|
|
2610
|
+
);
|
|
2611
|
+
const effectiveModelCache = cacheEnvironment.HF_HOME;
|
|
2612
|
+
const effectiveHubCache = resolveHuggingFaceCacheLayout(effectiveModelCache).hubCache;
|
|
2613
|
+
const env = withBundledPythonEnvironment(cacheEnvironment);
|
|
2614
|
+
const { exitCode } = await withProgressHeartbeat(args.reporter, downloadEvent, () => runLoggedProcess({
|
|
2615
|
+
command: entrypoint.command,
|
|
2616
|
+
commandArgs: entrypoint.commandArgs,
|
|
2617
|
+
env,
|
|
2618
|
+
logPath,
|
|
2619
|
+
// A base model can be several gigabytes. Surface the downloader's progress
|
|
2620
|
+
// by default; callers can still opt out by omitting the reporter (`--quiet`).
|
|
2621
|
+
reporter: args.reporter ? { ...args.reporter, verbose: true } : void 0,
|
|
2622
|
+
stage: "model_prefetch"
|
|
2623
|
+
}));
|
|
2624
|
+
if (exitCode !== 0) {
|
|
2625
|
+
throw new Error(`Model prefetch exited with code ${exitCode}. See ${logPath}.`);
|
|
2626
|
+
}
|
|
2627
|
+
const output = JSON.parse(await readFile3(outputPath, "utf8"));
|
|
2628
|
+
const snapshotRevision = baseModelRevisionSchema.safeParse(output.snapshot_revision);
|
|
2629
|
+
if (!snapshotRevision.success) {
|
|
2630
|
+
throw new Error(
|
|
2631
|
+
"Model prefetch did not resolve the base model to a 40-character immutable commit SHA."
|
|
2632
|
+
);
|
|
2633
|
+
}
|
|
2634
|
+
if (output.base_model !== payload.base_model) {
|
|
2635
|
+
throw new Error(`Model prefetch returned ${String(output.base_model)} for requested model ${payload.base_model}.`);
|
|
2636
|
+
}
|
|
2637
|
+
if (payload.revision && snapshotRevision.data !== payload.revision.toLowerCase()) {
|
|
2638
|
+
throw new Error(
|
|
2639
|
+
`Model prefetch resolved a different revision (${snapshotRevision.data}) than requested (${payload.revision}).`
|
|
2640
|
+
);
|
|
2641
|
+
}
|
|
2642
|
+
await verifyModelPrefetchCacheReport({
|
|
2643
|
+
modelId: payload.base_model,
|
|
2644
|
+
revision: snapshotRevision.data,
|
|
2645
|
+
modelCache: effectiveModelCache,
|
|
2646
|
+
output
|
|
2647
|
+
});
|
|
2648
|
+
if (defaultBaseModelRevision(payload.base_model)) {
|
|
2649
|
+
await withProgressHeartbeat(args.reporter, {
|
|
2650
|
+
stage: "model_prefetch",
|
|
2651
|
+
status: "running",
|
|
2652
|
+
message: "Checking model files against the certified revision's checksums."
|
|
2653
|
+
}, () => verifyLocalBaseModel(
|
|
2654
|
+
output.snapshot_path,
|
|
2655
|
+
payload.base_model,
|
|
2656
|
+
effectiveModelCache
|
|
2657
|
+
));
|
|
2658
|
+
}
|
|
2659
|
+
const recorded = args.localOnly ? void 0 : await withProgressHeartbeat(args.reporter, {
|
|
2660
|
+
stage: "model_prefetch",
|
|
2661
|
+
status: "running",
|
|
2662
|
+
message: "Verifying model files and recording the local asset manifest."
|
|
2663
|
+
}, () => recordAssetDownload({
|
|
2664
|
+
kind: "model",
|
|
2665
|
+
repo: payload.base_model,
|
|
2666
|
+
revision: snapshotRevision.data,
|
|
2667
|
+
snapshotPath: output.snapshot_path
|
|
2668
|
+
}));
|
|
2669
|
+
await args.reporter?.onEvent?.({
|
|
2670
|
+
stage: "model_prefetch",
|
|
2671
|
+
status: "completed",
|
|
2672
|
+
message: "Base model is available in the local Hugging Face cache.",
|
|
2673
|
+
details: {
|
|
2674
|
+
base_model: output.base_model ?? payload.base_model,
|
|
2675
|
+
hf_home: effectiveModelCache,
|
|
2676
|
+
hub_cache: effectiveHubCache,
|
|
2677
|
+
snapshot_path: output.snapshot_path ?? null,
|
|
2678
|
+
snapshot_revision: snapshotRevision.data,
|
|
2679
|
+
file_count: output.file_count ?? null,
|
|
2680
|
+
size_bytes: output.size_bytes ?? null,
|
|
2681
|
+
verified_blob_count: output.verified_blob_count ?? null
|
|
2682
|
+
}
|
|
2683
|
+
});
|
|
2684
|
+
return {
|
|
2685
|
+
ok: true,
|
|
2686
|
+
status: "completed",
|
|
2687
|
+
base_model: output.base_model ?? payload.base_model,
|
|
2688
|
+
model_cache: effectiveModelCache,
|
|
2689
|
+
hf_home: effectiveModelCache,
|
|
2690
|
+
hub_cache: effectiveHubCache,
|
|
2691
|
+
snapshot_path: output.snapshot_path,
|
|
2692
|
+
snapshot_revision: snapshotRevision.data,
|
|
2693
|
+
file_count: output.file_count,
|
|
2694
|
+
size_bytes: output.size_bytes,
|
|
2695
|
+
verified_blob_count: output.verified_blob_count,
|
|
2696
|
+
artifact_dir: artifactDir,
|
|
2697
|
+
input_path: inputPath,
|
|
2698
|
+
output_path: outputPath,
|
|
2699
|
+
log_uri: fileUri(logPath),
|
|
2700
|
+
command: entrypoint.displayCommand,
|
|
2701
|
+
...recorded ? { manifest_path: recorded.path } : {}
|
|
2702
|
+
};
|
|
2703
|
+
}
|
|
2704
|
+
|
|
2705
|
+
export {
|
|
2706
|
+
ARTIFACT_WORKFLOW_LOCK_FILE,
|
|
2707
|
+
fileUri,
|
|
2708
|
+
defaultArtifactPrefix,
|
|
2709
|
+
resolveRunArtifacts,
|
|
2710
|
+
claimRunArtifactDirectory,
|
|
2711
|
+
prepareRunDirectories,
|
|
2712
|
+
writeJson,
|
|
2713
|
+
writeJsonAtomic,
|
|
2714
|
+
writeFileAtomic,
|
|
2715
|
+
readJson,
|
|
2716
|
+
copyDatasetToTrainingChannel,
|
|
2717
|
+
writeArtifactManifest,
|
|
2718
|
+
assertArtifactManifest,
|
|
2719
|
+
TRAINING_MODELS,
|
|
2720
|
+
isCertifiedTrainingModel,
|
|
2721
|
+
resolveTrainingModel,
|
|
2722
|
+
resolveRequestedBaseModelRevision,
|
|
2723
|
+
defaultBaseModelRevision,
|
|
2724
|
+
resolveModelLoader,
|
|
2725
|
+
localModelArtifactPath,
|
|
2726
|
+
assertUsableModelArtifact,
|
|
2727
|
+
DECISION_MODELS,
|
|
2728
|
+
DEFAULT_DECISION_MODEL,
|
|
2729
|
+
resolveDecisionModel,
|
|
2730
|
+
DEFAULT_ARTIFACT_ROOT,
|
|
2731
|
+
DEFAULT_PROJECT_STORE_ROOT,
|
|
2732
|
+
DEFAULT_FOUNDATION_RUNS_DIR,
|
|
2733
|
+
DEFAULT_DECISION_RUNS_DIR,
|
|
2734
|
+
expandUserPath,
|
|
2735
|
+
getTunedTensorHome,
|
|
2736
|
+
getConfigDir,
|
|
2737
|
+
getAgentConfigDir,
|
|
2738
|
+
defaultStoreRoot,
|
|
2739
|
+
pythonEnvironmentPath,
|
|
2740
|
+
datasetFormatSchema,
|
|
2741
|
+
baseModelRevisionSchema,
|
|
2742
|
+
behaviorSpecExampleSchema,
|
|
2743
|
+
behaviorSpecSchema,
|
|
2744
|
+
specSnapshotSchema,
|
|
2745
|
+
foundationHyperparametersSchema,
|
|
2746
|
+
huggingFaceRepoIdSchema,
|
|
2747
|
+
datasetColumnsSchema,
|
|
2748
|
+
huggingFaceDatasetSourceSchema,
|
|
2749
|
+
isRepoRelativePath,
|
|
2750
|
+
datasetPrebuiltSchema,
|
|
2751
|
+
datasetSourceSchema,
|
|
2752
|
+
fineTuneHyperparametersSchema,
|
|
2753
|
+
fineTuneRunRequestSchema,
|
|
2754
|
+
evalExampleResultSchema,
|
|
2755
|
+
jsonFieldMetricsSchema,
|
|
2756
|
+
evalSplitSchema,
|
|
2757
|
+
evalReportSchema,
|
|
2758
|
+
comparisonReportSchema,
|
|
2759
|
+
trainingReportSchema,
|
|
2760
|
+
runProvenanceSchema,
|
|
2761
|
+
runReportSchema,
|
|
2762
|
+
localRunnerConfigSchema,
|
|
2763
|
+
projectRuntimeSchema,
|
|
2764
|
+
localFoundationSpecFileSchema,
|
|
2765
|
+
localAdapterSpecFileSchema,
|
|
2766
|
+
decisionQuestionSchema,
|
|
2767
|
+
decisionHyperparametersSchema,
|
|
2768
|
+
localDecisionSpecFileSchema,
|
|
2769
|
+
decisionLabels,
|
|
2770
|
+
localBehaviorSpecFileSchema,
|
|
2771
|
+
isFoundationSpecFile,
|
|
2772
|
+
isDecisionSpecFile,
|
|
2773
|
+
reportInBackground,
|
|
2774
|
+
withProgressHeartbeat,
|
|
2775
|
+
sanitizeLogLine,
|
|
2776
|
+
forwardStreamLines,
|
|
2777
|
+
localRuntimePackageRoot,
|
|
2778
|
+
createProgressRenderer,
|
|
2779
|
+
progressTty,
|
|
2780
|
+
progressColumns,
|
|
2781
|
+
buildDecisionPythonCommand,
|
|
2782
|
+
withDecisionPythonEnvironment,
|
|
2783
|
+
buildBundledPythonCommand,
|
|
2784
|
+
withBundledPythonEnvironment,
|
|
2785
|
+
buildFoundationPythonCommand,
|
|
2786
|
+
withFoundationPythonEnvironment,
|
|
2787
|
+
ProcessCancelledError,
|
|
2788
|
+
runLoggedProcess,
|
|
2789
|
+
minimalMachineLearningEnvironment,
|
|
2790
|
+
withHuggingFaceCacheEnvironment,
|
|
2791
|
+
withOfflineHuggingFaceCacheEnvironment,
|
|
2792
|
+
snapshotDigests,
|
|
2793
|
+
recordAssetDownload,
|
|
2794
|
+
listAssets,
|
|
2795
|
+
findAsset,
|
|
2796
|
+
verifyAsset,
|
|
2797
|
+
specLockPath,
|
|
2798
|
+
readSpecLock,
|
|
2799
|
+
writeSpecLockEntry,
|
|
2800
|
+
applySpecLock,
|
|
2801
|
+
enforceSpecLock,
|
|
2802
|
+
verifyModelPrefetchCacheReport,
|
|
2803
|
+
buildModelPrefetchPayload,
|
|
2804
|
+
buildModelSourcePrefetchPayload,
|
|
2805
|
+
safeName,
|
|
2806
|
+
verifyLocalBaseModel,
|
|
2807
|
+
prefetchBaseModel
|
|
2808
|
+
};
|
|
2809
|
+
//# sourceMappingURL=chunk-UCJCWJ6F.js.map
|