@tuned-tensor/cli 0.28.1 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -180
- package/dist/chunk-6CETSU2H.js +260 -0
- package/dist/chunk-6CETSU2H.js.map +1 -0
- package/dist/chunk-B2O6P7EX.js +1623 -0
- package/dist/chunk-B2O6P7EX.js.map +1 -0
- package/dist/{chunk-7CO5DBCY.js → chunk-GMBZ2FE6.js} +5046 -5014
- package/dist/chunk-GMBZ2FE6.js.map +1 -0
- package/dist/chunk-MM75IAY3.js +138 -0
- package/dist/chunk-MM75IAY3.js.map +1 -0
- package/dist/chunk-ST6ZSEUH.js +1272 -0
- package/dist/chunk-ST6ZSEUH.js.map +1 -0
- package/dist/contracts-SHBUXEOQ.js +72 -0
- package/dist/dataset-Y5NEUKI5.js +22 -0
- package/dist/dataset-Y5NEUKI5.js.map +1 -0
- package/dist/index.js +4530 -9349
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +84 -467
- package/dist/local-runtime.js.map +1 -1
- package/dist/model-server-IWCRFAOO.js +18 -0
- package/dist/model-server-IWCRFAOO.js.map +1 -0
- package/dist/{prefetch-LBIOO7ZO.js → prefetch-Q2ZA3SJ7.js} +3 -2
- package/dist/prefetch-Q2ZA3SJ7.js.map +1 -0
- package/docs/local-runtime/architecture.md +4 -4
- package/docs/local-runtime/aws-gpu.md +4 -5
- package/docs/local-runtime/decision.md +2 -2
- package/docs/local-runtime/hosting.md +103 -92
- package/docs/local-runtime/model-quality.md +103 -0
- package/docs/local-runtime/serving.md +1 -6
- package/docs/local-runtime/spark.md +2 -3
- package/docs/spec-architecture.html +2 -26
- package/docs/spec-workflow.md +9 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/LICENSE.md +55 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/README.md +364 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/index.d.ts +123 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/index.d.ts.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/index.js +778 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/index.js.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/index.min.js +3 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/index.min.js.map +7 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/numeric-range.d.ts +15 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/numeric-range.d.ts.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/numeric-range.js +76 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/numeric-range.js.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/package.json +3 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/parse-class.d.ts +4 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/parse-class.d.ts.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/parse-class.js +142 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/commonjs/parse-class.js.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/index.d.ts +123 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/index.d.ts.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/index.js +774 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/index.js.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/index.min.js +3 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/index.min.js.map +7 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/numeric-range.d.ts +15 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/numeric-range.d.ts.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/numeric-range.js +72 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/numeric-range.js.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/package.json +3 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/parse-class.d.ts +4 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/parse-class.d.ts.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/parse-class.js +138 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/dist/esm/parse-class.js.map +1 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/package.json +56 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/src/index.ts +932 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/src/numeric-range.ts +79 -0
- package/node_modules/@isaacs/nonrecursive-glob-match/src/parse-class.ts +158 -0
- package/node_modules/minimatch/README.md +125 -107
- package/node_modules/minimatch/dist/commonjs/index.d.ts +49 -38
- package/node_modules/minimatch/dist/commonjs/index.d.ts.map +1 -1
- package/node_modules/minimatch/dist/commonjs/index.js +782 -359
- package/node_modules/minimatch/dist/commonjs/index.js.map +1 -1
- package/node_modules/minimatch/dist/esm/index.d.ts +49 -38
- package/node_modules/minimatch/dist/esm/index.d.ts.map +1 -1
- package/node_modules/minimatch/dist/esm/index.js +781 -356
- package/node_modules/minimatch/dist/esm/index.js.map +1 -1
- package/node_modules/minimatch/package.json +5 -4
- package/package.json +4 -5
- package/training/adapter/src/train.py +16 -1
- package/dist/chunk-7CO5DBCY.js.map +0 -1
- package/dist/chunk-FSVTMX4L.js +0 -2820
- package/dist/chunk-FSVTMX4L.js.map +0 -1
- package/node_modules/minimatch/dist/commonjs/ast.d.ts +0 -22
- package/node_modules/minimatch/dist/commonjs/ast.d.ts.map +0 -1
- package/node_modules/minimatch/dist/commonjs/ast.js +0 -845
- package/node_modules/minimatch/dist/commonjs/ast.js.map +0 -1
- package/node_modules/minimatch/dist/commonjs/brace-expressions.d.ts +0 -8
- package/node_modules/minimatch/dist/commonjs/brace-expressions.d.ts.map +0 -1
- package/node_modules/minimatch/dist/commonjs/brace-expressions.js +0 -150
- package/node_modules/minimatch/dist/commonjs/brace-expressions.js.map +0 -1
- package/node_modules/minimatch/dist/esm/ast.d.ts +0 -22
- package/node_modules/minimatch/dist/esm/ast.d.ts.map +0 -1
- package/node_modules/minimatch/dist/esm/ast.js +0 -841
- package/node_modules/minimatch/dist/esm/ast.js.map +0 -1
- package/node_modules/minimatch/dist/esm/brace-expressions.d.ts +0 -8
- package/node_modules/minimatch/dist/esm/brace-expressions.d.ts.map +0 -1
- package/node_modules/minimatch/dist/esm/brace-expressions.js +0 -146
- package/node_modules/minimatch/dist/esm/brace-expressions.js.map +0 -1
- /package/dist/{prefetch-LBIOO7ZO.js.map → contracts-SHBUXEOQ.js.map} +0 -0
|
@@ -0,0 +1,1272 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
assertCertifiedBaseModelConfig,
|
|
4
|
+
baseModelRevisionSchema,
|
|
5
|
+
defaultBaseModelRevision,
|
|
6
|
+
fileUri,
|
|
7
|
+
getTunedTensorHome,
|
|
8
|
+
isCertifiedTrainingModel,
|
|
9
|
+
pythonEnvironmentPath,
|
|
10
|
+
resolveRequestedBaseModelRevision,
|
|
11
|
+
resolveTrainingModel,
|
|
12
|
+
writeJson,
|
|
13
|
+
writeJsonAtomic
|
|
14
|
+
} from "./chunk-B2O6P7EX.js";
|
|
15
|
+
|
|
16
|
+
// src/local-runtime/prefetch.ts
|
|
17
|
+
import { createHash as createHash3 } from "crypto";
|
|
18
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
19
|
+
import { lstat, mkdir as mkdir2, readFile as readFile2, readdir as readdir2, realpath as realpath2, stat as stat2 } from "fs/promises";
|
|
20
|
+
import { isAbsolute, join as join4, relative as relative2, resolve as resolve4, sep as sep2 } from "path";
|
|
21
|
+
|
|
22
|
+
// src/local-runtime/process-runner.ts
|
|
23
|
+
import { createHash } from "crypto";
|
|
24
|
+
import { spawn } from "child_process";
|
|
25
|
+
import { constants, readFileSync } from "fs";
|
|
26
|
+
import { mkdir, open } from "fs/promises";
|
|
27
|
+
import { dirname as dirname2, join, resolve as resolve2 } from "path";
|
|
28
|
+
|
|
29
|
+
// src/local-runtime/run-reporter.ts
|
|
30
|
+
function reportInBackground(action) {
|
|
31
|
+
try {
|
|
32
|
+
void Promise.resolve(action()).catch(() => void 0);
|
|
33
|
+
} catch {
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
async function withProgressHeartbeat(reporter, event, run) {
|
|
37
|
+
if (!reporter?.onEvent) return await run();
|
|
38
|
+
reportInBackground(() => reporter.onEvent(event));
|
|
39
|
+
const timer = setInterval(() => {
|
|
40
|
+
reportInBackground(() => reporter.onEvent({
|
|
41
|
+
stage: event.stage,
|
|
42
|
+
status: event.status,
|
|
43
|
+
message: `Still working: ${event.message}`
|
|
44
|
+
}));
|
|
45
|
+
}, 15e3);
|
|
46
|
+
timer.unref();
|
|
47
|
+
try {
|
|
48
|
+
return await run();
|
|
49
|
+
} finally {
|
|
50
|
+
clearInterval(timer);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
function sanitizeLogLine(line) {
|
|
54
|
+
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
55
|
+
}
|
|
56
|
+
function forwardStreamLines(stream, onLine) {
|
|
57
|
+
let buffered = "";
|
|
58
|
+
stream.setEncoding("utf8");
|
|
59
|
+
stream.on("data", (chunk) => {
|
|
60
|
+
buffered += chunk;
|
|
61
|
+
const lines = buffered.split(/\r\n|\n|\r/);
|
|
62
|
+
buffered = lines.pop() ?? "";
|
|
63
|
+
for (const line of lines) {
|
|
64
|
+
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
stream.on("end", () => {
|
|
68
|
+
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// src/local-runtime/package-root.ts
|
|
73
|
+
import { existsSync } from "fs";
|
|
74
|
+
import { dirname, resolve } from "path";
|
|
75
|
+
import { fileURLToPath } from "url";
|
|
76
|
+
function localRuntimePackageRoot(moduleUrl) {
|
|
77
|
+
const moduleDirectory = dirname(fileURLToPath(moduleUrl));
|
|
78
|
+
const candidates = [
|
|
79
|
+
dirname(moduleDirectory),
|
|
80
|
+
dirname(dirname(moduleDirectory))
|
|
81
|
+
];
|
|
82
|
+
const root = candidates.find(
|
|
83
|
+
(candidate) => existsSync(resolve(candidate, "package.json")) && existsSync(resolve(candidate, "training/adapter"))
|
|
84
|
+
);
|
|
85
|
+
if (!root) {
|
|
86
|
+
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
87
|
+
}
|
|
88
|
+
return root;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// src/local-runtime/progress-bar.ts
|
|
92
|
+
var PROGRESS_PREFIX = "@@tt-progress ";
|
|
93
|
+
function nonNegativeInteger(value) {
|
|
94
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : null;
|
|
95
|
+
}
|
|
96
|
+
function parseProgressLine(line) {
|
|
97
|
+
const start = line.indexOf(PROGRESS_PREFIX);
|
|
98
|
+
if (start !== 0) return null;
|
|
99
|
+
let value;
|
|
100
|
+
try {
|
|
101
|
+
value = JSON.parse(line.slice(PROGRESS_PREFIX.length));
|
|
102
|
+
} catch {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
106
|
+
const record = value;
|
|
107
|
+
const completed = nonNegativeInteger(record.completed_bytes);
|
|
108
|
+
const total = nonNegativeInteger(record.total_bytes);
|
|
109
|
+
if (completed === null || total === null) return null;
|
|
110
|
+
return {
|
|
111
|
+
label: typeof record.label === "string" ? record.label : "download",
|
|
112
|
+
phase: typeof record.phase === "string" ? record.phase : "download",
|
|
113
|
+
completed_bytes: completed,
|
|
114
|
+
total_bytes: total,
|
|
115
|
+
files_completed: nonNegativeInteger(record.files_completed) ?? 0,
|
|
116
|
+
files_total: nonNegativeInteger(record.files_total) ?? 0
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
function formatBytes(bytes) {
|
|
120
|
+
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
121
|
+
let value = bytes;
|
|
122
|
+
let unit = 0;
|
|
123
|
+
while (value >= 1024 && unit < units.length - 1) {
|
|
124
|
+
value /= 1024;
|
|
125
|
+
unit += 1;
|
|
126
|
+
}
|
|
127
|
+
return `${unit === 0 ? value.toFixed(0) : value.toFixed(value >= 100 ? 0 : value >= 10 ? 1 : 2)} ${units[unit]}`;
|
|
128
|
+
}
|
|
129
|
+
function formatDuration(seconds) {
|
|
130
|
+
if (!Number.isFinite(seconds) || seconds < 0) return "--";
|
|
131
|
+
const whole = Math.round(seconds);
|
|
132
|
+
const hours = Math.floor(whole / 3600);
|
|
133
|
+
const minutes = Math.floor(whole % 3600 / 60);
|
|
134
|
+
const rest = whole % 60;
|
|
135
|
+
if (hours > 0) return `${hours}h ${String(minutes).padStart(2, "0")}m`;
|
|
136
|
+
if (minutes > 0) return `${minutes}m ${String(rest).padStart(2, "0")}s`;
|
|
137
|
+
return `${rest}s`;
|
|
138
|
+
}
|
|
139
|
+
var RateTracker = class {
|
|
140
|
+
constructor(windowMs = 15e3) {
|
|
141
|
+
this.windowMs = windowMs;
|
|
142
|
+
}
|
|
143
|
+
windowMs;
|
|
144
|
+
samples = [];
|
|
145
|
+
add(at, bytes) {
|
|
146
|
+
const last = this.samples.at(-1);
|
|
147
|
+
if (last && bytes < last.bytes) this.samples = [];
|
|
148
|
+
this.samples.push({ at, bytes });
|
|
149
|
+
while (this.samples.length > 2 && at - this.samples[0].at > this.windowMs) this.samples.shift();
|
|
150
|
+
}
|
|
151
|
+
bytesPerSecond() {
|
|
152
|
+
const first = this.samples[0];
|
|
153
|
+
const last = this.samples.at(-1);
|
|
154
|
+
if (!first || !last || last.at - first.at < 500) return null;
|
|
155
|
+
return (last.bytes - first.bytes) / ((last.at - first.at) / 1e3);
|
|
156
|
+
}
|
|
157
|
+
};
|
|
158
|
+
function formatProgressLine(progress, rate, columns) {
|
|
159
|
+
const total = progress.total_bytes;
|
|
160
|
+
const fraction = total > 0 ? Math.min(1, progress.completed_bytes / total) : 1;
|
|
161
|
+
const fields = [
|
|
162
|
+
{ text: `${(fraction * 100).toFixed(1).padStart(5)}%` },
|
|
163
|
+
{ text: `${formatBytes(progress.completed_bytes)} / ${formatBytes(total)}` }
|
|
164
|
+
];
|
|
165
|
+
const done = progress.completed_bytes >= total;
|
|
166
|
+
if (!done && rate !== null && rate > 0) {
|
|
167
|
+
fields.push(
|
|
168
|
+
{ text: `${formatBytes(rate)}/s`, drop: 2 },
|
|
169
|
+
{ text: `ETA ${formatDuration((total - progress.completed_bytes) / rate)}`, drop: 3 }
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
if (progress.files_total > 0) fields.push({ text: `${progress.files_completed}/${progress.files_total} files`, drop: 1 });
|
|
173
|
+
const prefix = `[tt] ${progress.label}: `;
|
|
174
|
+
const join5 = () => fields.map((field) => field.text).join(" ");
|
|
175
|
+
if (columns === void 0) return `${prefix}${join5()}`;
|
|
176
|
+
for (const rank of [1, 2, 3]) {
|
|
177
|
+
if (prefix.length + join5().length <= columns - 1) break;
|
|
178
|
+
const index = fields.findIndex((field) => field.drop === rank);
|
|
179
|
+
if (index !== -1) fields.splice(index, 1);
|
|
180
|
+
}
|
|
181
|
+
const text = join5();
|
|
182
|
+
const barWidth = Math.min(30, columns - 1 - prefix.length - text.length - 3);
|
|
183
|
+
if (barWidth < 10) return `${prefix}${text}`.slice(0, Math.max(0, columns - 1));
|
|
184
|
+
const filled = Math.round(fraction * barWidth);
|
|
185
|
+
return `${prefix}[${"#".repeat(filled)}${"-".repeat(barWidth - filled)}] ${text}`;
|
|
186
|
+
}
|
|
187
|
+
function createProgressRenderer(options) {
|
|
188
|
+
const now = options.now ?? (() => Date.now());
|
|
189
|
+
const rates = /* @__PURE__ */ new Map();
|
|
190
|
+
const lastLine = /* @__PURE__ */ new Map();
|
|
191
|
+
const completed = /* @__PURE__ */ new Set();
|
|
192
|
+
let drawn = false;
|
|
193
|
+
let lastDraw = Number.NEGATIVE_INFINITY;
|
|
194
|
+
const finished = (progress) => progress.phase === "downloaded" || progress.total_bytes > 0 && progress.completed_bytes >= progress.total_bytes;
|
|
195
|
+
return {
|
|
196
|
+
update(progress) {
|
|
197
|
+
const at = now();
|
|
198
|
+
const tracker = rates.get(progress.label) ?? new RateTracker();
|
|
199
|
+
rates.set(progress.label, tracker);
|
|
200
|
+
tracker.add(at, progress.completed_bytes);
|
|
201
|
+
const done = finished(progress);
|
|
202
|
+
if (options.tty) {
|
|
203
|
+
if (done && completed.has(progress.label)) return;
|
|
204
|
+
if (!done) completed.delete(progress.label);
|
|
205
|
+
if (!done && at - lastDraw < 100) return;
|
|
206
|
+
lastDraw = at;
|
|
207
|
+
const line = formatProgressLine(progress, tracker.bytesPerSecond(), Math.max(40, options.columns ?? 80));
|
|
208
|
+
options.write(`\r\x1B[2K${line}${done ? "\n" : ""}`);
|
|
209
|
+
drawn = !done;
|
|
210
|
+
if (done) {
|
|
211
|
+
rates.delete(progress.label);
|
|
212
|
+
completed.add(progress.label);
|
|
213
|
+
}
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
const fraction = progress.total_bytes > 0 ? progress.completed_bytes / progress.total_bytes : 1;
|
|
217
|
+
const decile = Math.floor(fraction * 10);
|
|
218
|
+
const previous = lastLine.get(progress.label);
|
|
219
|
+
const interval = options.lineIntervalMs ?? 3e4;
|
|
220
|
+
if (previous && !done && decile <= previous.decile && at - previous.at < interval) return;
|
|
221
|
+
if (previous && done && previous.decile === 10) return;
|
|
222
|
+
lastLine.set(progress.label, { at, decile: done ? 10 : decile });
|
|
223
|
+
options.write(`${formatProgressLine(progress, tracker.bytesPerSecond())}
|
|
224
|
+
`);
|
|
225
|
+
},
|
|
226
|
+
interrupt() {
|
|
227
|
+
if (!drawn) return;
|
|
228
|
+
options.write("\r\x1B[2K");
|
|
229
|
+
drawn = false;
|
|
230
|
+
}
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
function progressTty(env = process.env, stream = process.stderr) {
|
|
234
|
+
if (env.TT_PROGRESS_TTY === "0") return false;
|
|
235
|
+
return env.TT_PROGRESS_TTY === "1" || Boolean(stream.isTTY);
|
|
236
|
+
}
|
|
237
|
+
function progressColumns(env = process.env, stream = process.stderr) {
|
|
238
|
+
const hinted = Number(env.TT_PROGRESS_COLUMNS);
|
|
239
|
+
if (Number.isInteger(hinted) && hinted > 0) return hinted;
|
|
240
|
+
return stream.columns;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
// src/local-runtime/process-runner.ts
|
|
244
|
+
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
245
|
+
var bundledProject = join(packageRoot, "training/adapter");
|
|
246
|
+
var bundledRuntimeHash = (() => {
|
|
247
|
+
const hash = createHash("sha256");
|
|
248
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
249
|
+
hash.update(readFileSync(join(bundledProject, name)));
|
|
250
|
+
}
|
|
251
|
+
return hash.digest("hex").slice(0, 20);
|
|
252
|
+
})();
|
|
253
|
+
var BUNDLED_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv", bundledRuntimeHash);
|
|
254
|
+
var foundationProject = join(packageRoot, "training/foundation");
|
|
255
|
+
var foundationRuntimeHash = (() => {
|
|
256
|
+
const hash = createHash("sha256");
|
|
257
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
258
|
+
hash.update(readFileSync(join(foundationProject, name)));
|
|
259
|
+
}
|
|
260
|
+
return hash.digest("hex").slice(0, 20);
|
|
261
|
+
})();
|
|
262
|
+
var FOUNDATION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-foundation", foundationRuntimeHash);
|
|
263
|
+
var decisionProject = join(packageRoot, "training/decision");
|
|
264
|
+
var decisionRuntimeHash = (() => {
|
|
265
|
+
const hash = createHash("sha256");
|
|
266
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
267
|
+
hash.update(readFileSync(join(decisionProject, name)));
|
|
268
|
+
}
|
|
269
|
+
return hash.digest("hex").slice(0, 20);
|
|
270
|
+
})();
|
|
271
|
+
var DECISION_PYTHON_ENVIRONMENT = pythonEnvironmentPath("uv-decision", decisionRuntimeHash);
|
|
272
|
+
function buildDecisionPythonCommand(entrypoint, args = []) {
|
|
273
|
+
const commandArgs = [
|
|
274
|
+
"run",
|
|
275
|
+
"--frozen",
|
|
276
|
+
"--quiet",
|
|
277
|
+
"--project",
|
|
278
|
+
decisionProject,
|
|
279
|
+
"python",
|
|
280
|
+
entrypoint === "-c" ? entrypoint : join(decisionProject, "src", entrypoint),
|
|
281
|
+
...args
|
|
282
|
+
];
|
|
283
|
+
return { command: "uv", commandArgs, displayCommand: ["uv", ...commandArgs] };
|
|
284
|
+
}
|
|
285
|
+
function withDecisionPythonEnvironment(env) {
|
|
286
|
+
return { ...env, UV_PROJECT_ENVIRONMENT: DECISION_PYTHON_ENVIRONMENT };
|
|
287
|
+
}
|
|
288
|
+
function buildBundledPythonCommand(entrypoint, args = []) {
|
|
289
|
+
const target = entrypoint === "-c" ? entrypoint : join(bundledProject, "src", entrypoint);
|
|
290
|
+
const commandArgs = [
|
|
291
|
+
"run",
|
|
292
|
+
"--frozen",
|
|
293
|
+
"--quiet",
|
|
294
|
+
"--project",
|
|
295
|
+
bundledProject,
|
|
296
|
+
"python",
|
|
297
|
+
target,
|
|
298
|
+
...args
|
|
299
|
+
];
|
|
300
|
+
return {
|
|
301
|
+
command: "uv",
|
|
302
|
+
commandArgs,
|
|
303
|
+
displayCommand: ["uv", ...commandArgs]
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
function withBundledPythonEnvironment(env) {
|
|
307
|
+
return {
|
|
308
|
+
...env,
|
|
309
|
+
UV_PROJECT_ENVIRONMENT: BUNDLED_PYTHON_ENVIRONMENT
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
function buildFoundationPythonCommand(entrypoint, args = []) {
|
|
313
|
+
const target = entrypoint === "-c" ? entrypoint : join(foundationProject, "src", entrypoint);
|
|
314
|
+
const commandArgs = [
|
|
315
|
+
"run",
|
|
316
|
+
"--frozen",
|
|
317
|
+
"--quiet",
|
|
318
|
+
"--project",
|
|
319
|
+
foundationProject,
|
|
320
|
+
"python",
|
|
321
|
+
target,
|
|
322
|
+
...args
|
|
323
|
+
];
|
|
324
|
+
return {
|
|
325
|
+
command: "uv",
|
|
326
|
+
commandArgs,
|
|
327
|
+
displayCommand: ["uv", ...commandArgs]
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
function withFoundationPythonEnvironment(env) {
|
|
331
|
+
return {
|
|
332
|
+
...env,
|
|
333
|
+
UV_PROJECT_ENVIRONMENT: FOUNDATION_PYTHON_ENVIRONMENT
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
var ProcessCancelledError = class extends Error {
|
|
337
|
+
constructor(message = "Process cancelled.") {
|
|
338
|
+
super(message);
|
|
339
|
+
this.name = "ProcessCancelledError";
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
async function openProcessLog(path, exclusive, append) {
|
|
343
|
+
const flags = constants.O_WRONLY | constants.O_CREAT | (exclusive ? constants.O_EXCL : append ? constants.O_APPEND : constants.O_TRUNC) | (process.platform === "win32" ? 0 : constants.O_NOFOLLOW);
|
|
344
|
+
const handle = await open(path, flags, 384);
|
|
345
|
+
try {
|
|
346
|
+
return handle.createWriteStream({ autoClose: true });
|
|
347
|
+
} catch (error) {
|
|
348
|
+
await handle.close();
|
|
349
|
+
throw error;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
async function runLoggedProcess(args) {
|
|
353
|
+
if (args.logPath) await mkdir(dirname2(args.logPath), { recursive: true });
|
|
354
|
+
const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false, args.appendLog ?? false) : null;
|
|
355
|
+
let stderr = "";
|
|
356
|
+
try {
|
|
357
|
+
const exitCode = await new Promise((resolvePromise, reject) => {
|
|
358
|
+
const child = spawn(args.command, args.commandArgs, {
|
|
359
|
+
cwd: args.cwd ? resolve2(args.cwd) : process.cwd(),
|
|
360
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
361
|
+
env: args.env ?? process.env,
|
|
362
|
+
detached: process.platform !== "win32"
|
|
363
|
+
});
|
|
364
|
+
let timedOut = false;
|
|
365
|
+
let cancelled = false;
|
|
366
|
+
let cancellationError;
|
|
367
|
+
let cancellationCheckRunning = false;
|
|
368
|
+
let forceKillTimer = null;
|
|
369
|
+
const killProcessGroup = (signal) => {
|
|
370
|
+
if (child.pid && process.platform !== "win32") {
|
|
371
|
+
try {
|
|
372
|
+
process.kill(-child.pid, signal);
|
|
373
|
+
return;
|
|
374
|
+
} catch {
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
child.kill(signal);
|
|
378
|
+
};
|
|
379
|
+
const requestStop = (signal = "SIGTERM") => {
|
|
380
|
+
killProcessGroup(signal);
|
|
381
|
+
if (!forceKillTimer) {
|
|
382
|
+
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), args.shutdownGraceMs ?? 5e3);
|
|
383
|
+
forceKillTimer.unref();
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
const onSigint = () => {
|
|
387
|
+
cancelled = true;
|
|
388
|
+
requestStop("SIGINT");
|
|
389
|
+
};
|
|
390
|
+
const onSigterm = () => {
|
|
391
|
+
cancelled = true;
|
|
392
|
+
requestStop("SIGTERM");
|
|
393
|
+
};
|
|
394
|
+
process.once("SIGINT", onSigint);
|
|
395
|
+
process.once("SIGTERM", onSigterm);
|
|
396
|
+
const timer = args.timeoutMs ? setTimeout(() => {
|
|
397
|
+
timedOut = true;
|
|
398
|
+
requestStop();
|
|
399
|
+
}, args.timeoutMs) : null;
|
|
400
|
+
const cancellationTimer = args.shouldCancel ? setInterval(() => {
|
|
401
|
+
if (cancellationCheckRunning || cancelled) return;
|
|
402
|
+
cancellationCheckRunning = true;
|
|
403
|
+
Promise.resolve(args.shouldCancel?.()).then((requested) => {
|
|
404
|
+
if (!requested || cancelled) return;
|
|
405
|
+
cancelled = true;
|
|
406
|
+
requestStop();
|
|
407
|
+
}).catch((error) => {
|
|
408
|
+
cancellationError = error;
|
|
409
|
+
requestStop();
|
|
410
|
+
}).finally(() => {
|
|
411
|
+
cancellationCheckRunning = false;
|
|
412
|
+
});
|
|
413
|
+
}, args.cancelPollMs ?? 250) : null;
|
|
414
|
+
cancellationTimer?.unref();
|
|
415
|
+
const clearProcessTimers = () => {
|
|
416
|
+
if (timer) clearTimeout(timer);
|
|
417
|
+
if (cancellationTimer) clearInterval(cancellationTimer);
|
|
418
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
419
|
+
process.off("SIGINT", onSigint);
|
|
420
|
+
process.off("SIGTERM", onSigterm);
|
|
421
|
+
};
|
|
422
|
+
if (logStream) {
|
|
423
|
+
child.stdout.pipe(logStream, { end: false });
|
|
424
|
+
child.stderr.pipe(logStream, { end: false });
|
|
425
|
+
}
|
|
426
|
+
forwardStreamLines(child.stdout, (line) => {
|
|
427
|
+
const progress = parseProgressLine(line);
|
|
428
|
+
if (progress) {
|
|
429
|
+
reportInBackground(() => args.reporter?.onProgress?.(progress));
|
|
430
|
+
return;
|
|
431
|
+
}
|
|
432
|
+
args.onLine?.(line, "stdout");
|
|
433
|
+
if (args.reporter?.verbose) {
|
|
434
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stdout", message: line }));
|
|
435
|
+
}
|
|
436
|
+
});
|
|
437
|
+
forwardStreamLines(child.stderr, (line) => {
|
|
438
|
+
stderr += `${line}
|
|
439
|
+
`;
|
|
440
|
+
args.onLine?.(line, "stderr");
|
|
441
|
+
if (args.reporter?.verbose) {
|
|
442
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stderr", message: line }));
|
|
443
|
+
}
|
|
444
|
+
});
|
|
445
|
+
child.on("error", (error) => {
|
|
446
|
+
clearProcessTimers();
|
|
447
|
+
reject(timedOut ? new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`) : error);
|
|
448
|
+
});
|
|
449
|
+
child.on("close", (code) => {
|
|
450
|
+
if (timedOut || cancelled || cancellationError || args.terminateProcessGroupOnExit) {
|
|
451
|
+
killProcessGroup("SIGKILL");
|
|
452
|
+
}
|
|
453
|
+
clearProcessTimers();
|
|
454
|
+
if (timedOut) {
|
|
455
|
+
reject(new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`));
|
|
456
|
+
return;
|
|
457
|
+
}
|
|
458
|
+
if (cancellationError) {
|
|
459
|
+
reject(cancellationError);
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
if (cancelled) {
|
|
463
|
+
reject(new ProcessCancelledError(`${args.stage} was cancelled.`));
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
resolvePromise(code ?? 1);
|
|
467
|
+
});
|
|
468
|
+
});
|
|
469
|
+
return { exitCode, stderr };
|
|
470
|
+
} finally {
|
|
471
|
+
if (logStream) {
|
|
472
|
+
await new Promise((resolveEnd) => {
|
|
473
|
+
logStream.end(resolveEnd);
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// src/local-runtime/assets.ts
|
|
480
|
+
import { createHash as createHash2 } from "crypto";
|
|
481
|
+
import { createReadStream } from "fs";
|
|
482
|
+
import { readFile, readdir, realpath, stat } from "fs/promises";
|
|
483
|
+
import { basename, dirname as dirname4, join as join3, relative, sep } from "path";
|
|
484
|
+
|
|
485
|
+
// src/local-runtime/huggingface-cache.ts
|
|
486
|
+
import { homedir } from "os";
|
|
487
|
+
import { dirname as dirname3, join as join2, resolve as resolve3 } from "path";
|
|
488
|
+
function minimalMachineLearningEnvironment(env) {
|
|
489
|
+
const exact = /* @__PURE__ */ new Set([
|
|
490
|
+
"PATH",
|
|
491
|
+
"HOME",
|
|
492
|
+
"USER",
|
|
493
|
+
"LOGNAME",
|
|
494
|
+
"SHELL",
|
|
495
|
+
"TMPDIR",
|
|
496
|
+
"TEMP",
|
|
497
|
+
"TMP",
|
|
498
|
+
"LANG",
|
|
499
|
+
"LANGUAGE",
|
|
500
|
+
"LD_LIBRARY_PATH",
|
|
501
|
+
"DYLD_LIBRARY_PATH",
|
|
502
|
+
"XDG_CACHE_HOME",
|
|
503
|
+
"SSL_CERT_FILE",
|
|
504
|
+
"REQUESTS_CA_BUNDLE",
|
|
505
|
+
"CURL_CA_BUNDLE",
|
|
506
|
+
"NO_PROXY",
|
|
507
|
+
"no_proxy",
|
|
508
|
+
"UV_CACHE_DIR",
|
|
509
|
+
"UV_PYTHON",
|
|
510
|
+
"HF_HOME",
|
|
511
|
+
"HF_HUB_CACHE",
|
|
512
|
+
"HUGGINGFACE_HUB_CACHE",
|
|
513
|
+
"HF_ENDPOINT",
|
|
514
|
+
"HF_TOKEN",
|
|
515
|
+
"HF_HUB_OFFLINE",
|
|
516
|
+
"HF_HUB_DISABLE_TELEMETRY",
|
|
517
|
+
"TRANSFORMERS_OFFLINE",
|
|
518
|
+
"HF_HUB_ENABLE_HF_TRANSFER",
|
|
519
|
+
"HF_HUB_DOWNLOAD_TIMEOUT",
|
|
520
|
+
"HF_HUB_ETAG_TIMEOUT",
|
|
521
|
+
"TT_HF_DOWNLOAD_MAX_ATTEMPTS",
|
|
522
|
+
"PYTHONUTF8",
|
|
523
|
+
"PYTHONIOENCODING"
|
|
524
|
+
]);
|
|
525
|
+
const prefixes = [
|
|
526
|
+
"LC_",
|
|
527
|
+
"CUDA_",
|
|
528
|
+
"NVIDIA_",
|
|
529
|
+
"NCCL_",
|
|
530
|
+
"OMP_",
|
|
531
|
+
"MKL_",
|
|
532
|
+
"TORCH_",
|
|
533
|
+
"PYTORCH_",
|
|
534
|
+
"TOKENIZERS_",
|
|
535
|
+
"HF_XET_"
|
|
536
|
+
];
|
|
537
|
+
const result = Object.fromEntries(Object.entries(env).filter(
|
|
538
|
+
([key]) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix))
|
|
539
|
+
));
|
|
540
|
+
result.TORCH_DISABLE_NATIVE_JIT ??= "1";
|
|
541
|
+
return result;
|
|
542
|
+
}
|
|
543
|
+
function resolveHuggingFaceCacheLayout(modelCache) {
|
|
544
|
+
const hfHome = resolve3(modelCache);
|
|
545
|
+
return {
|
|
546
|
+
hfHome,
|
|
547
|
+
hubCache: join2(hfHome, "hub")
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
function withHuggingFaceCacheEnvironment(env, modelCache) {
|
|
551
|
+
const inheritedHub = env.HF_HUB_CACHE ?? env.HUGGINGFACE_HUB_CACHE;
|
|
552
|
+
const canonicalHome = modelCache ?? env.HF_HOME ?? (inheritedHub ? dirname3(resolve3(inheritedHub)) : join2(homedir(), ".cache", "huggingface"));
|
|
553
|
+
const layout = resolveHuggingFaceCacheLayout(canonicalHome);
|
|
554
|
+
const result = {
|
|
555
|
+
...env,
|
|
556
|
+
HF_HOME: layout.hfHome,
|
|
557
|
+
HF_HUB_CACHE: layout.hubCache,
|
|
558
|
+
HUGGINGFACE_HUB_CACHE: layout.hubCache
|
|
559
|
+
};
|
|
560
|
+
delete result.TRANSFORMERS_CACHE;
|
|
561
|
+
delete result.PYTORCH_TRANSFORMERS_CACHE;
|
|
562
|
+
delete result.PYTORCH_PRETRAINED_BERT_CACHE;
|
|
563
|
+
return result;
|
|
564
|
+
}
|
|
565
|
+
function withOfflineHuggingFaceCacheEnvironment(env, modelCache) {
|
|
566
|
+
return {
|
|
567
|
+
...withHuggingFaceCacheEnvironment(env, modelCache),
|
|
568
|
+
HF_HUB_OFFLINE: "1",
|
|
569
|
+
TRANSFORMERS_OFFLINE: "1"
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// src/local-runtime/assets.ts
|
|
574
|
+
var COMMIT_SHA = /^[0-9a-f]{40}$/;
|
|
575
|
+
var SpecLockMismatchError = class extends Error {
|
|
576
|
+
constructor(message) {
|
|
577
|
+
super(message);
|
|
578
|
+
this.name = "SpecLockMismatchError";
|
|
579
|
+
}
|
|
580
|
+
};
|
|
581
|
+
function assetsRoot(env = process.env) {
|
|
582
|
+
return join3(getTunedTensorHome(env), "assets");
|
|
583
|
+
}
|
|
584
|
+
function snapshotDirectory(hubCache, kind, repo, revision) {
|
|
585
|
+
return join3(hubCache, `${kind}s--${repo.replaceAll("/", "--")}`, "snapshots", revision);
|
|
586
|
+
}
|
|
587
|
+
async function contentDigest(path, algorithm) {
|
|
588
|
+
const size = (await stat(path)).size;
|
|
589
|
+
const hash = algorithm === "sha256" ? createHash2("sha256") : createHash2("sha1").update(`blob ${size}\0`);
|
|
590
|
+
for await (const chunk of createReadStream(path, { highWaterMark: 8 * 1024 * 1024 })) hash.update(chunk);
|
|
591
|
+
return `${algorithm}:${hash.digest("hex")}`;
|
|
592
|
+
}
|
|
593
|
+
async function listFiles(root, directory = root) {
|
|
594
|
+
const files = [];
|
|
595
|
+
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
|
596
|
+
const child = join3(directory, entry.name);
|
|
597
|
+
if (entry.isDirectory()) files.push(...await listFiles(root, child));
|
|
598
|
+
else files.push(relative(root, child).split(sep).join("/"));
|
|
599
|
+
}
|
|
600
|
+
return files.sort();
|
|
601
|
+
}
|
|
602
|
+
async function snapshotDigests(snapshot, paths) {
|
|
603
|
+
const files = {};
|
|
604
|
+
let sizeBytes = 0;
|
|
605
|
+
for (const path of paths ?? await listFiles(snapshot)) {
|
|
606
|
+
const local = join3(snapshot, ...path.split("/"));
|
|
607
|
+
const target = await realpath(local);
|
|
608
|
+
sizeBytes += (await stat(target)).size;
|
|
609
|
+
const blob = target !== local && basename(dirname4(target)) === "blobs" ? basename(target).toLowerCase() : "";
|
|
610
|
+
files[path] = await contentDigest(target, /^[0-9a-f]{40}$/.test(blob) ? "git-sha1" : "sha256");
|
|
611
|
+
if (/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/.test(blob) && files[path].split(":")[1] !== blob) {
|
|
612
|
+
throw new SpecLockMismatchError(`Cached file ${path} failed its checksum. Remove the corrupt blob ${target} and prefetch it again.`);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
return { files, sizeBytes };
|
|
616
|
+
}
|
|
617
|
+
function manifestPath(kind, repo, revision, env) {
|
|
618
|
+
return join3(assetsRoot(env), `${kind}s`, repo.replaceAll("/", "--"), revision, "manifest.json");
|
|
619
|
+
}
|
|
620
|
+
async function recordAssetDownload(args) {
|
|
621
|
+
const path = manifestPath(args.kind, args.repo, args.revision, args.env);
|
|
622
|
+
let paths = args.paths;
|
|
623
|
+
if (paths) {
|
|
624
|
+
const previous = Object.keys((await readJsonIfPresent(path))?.files ?? {});
|
|
625
|
+
const stillCached = await Promise.all(previous.map(async (file) => await stat(join3(args.snapshotPath, ...file.split("/"))).then(() => file, () => null)));
|
|
626
|
+
paths = [.../* @__PURE__ */ new Set([...stillCached.filter((file) => file !== null), ...paths])].sort();
|
|
627
|
+
}
|
|
628
|
+
const { files, sizeBytes } = await snapshotDigests(args.snapshotPath, paths);
|
|
629
|
+
const manifest = {
|
|
630
|
+
version: 1,
|
|
631
|
+
kind: args.kind,
|
|
632
|
+
repo: args.repo,
|
|
633
|
+
revision: args.revision,
|
|
634
|
+
fetched_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
635
|
+
snapshot_path: args.snapshotPath,
|
|
636
|
+
size_bytes: sizeBytes,
|
|
637
|
+
files
|
|
638
|
+
};
|
|
639
|
+
await writeJsonAtomic(path, manifest);
|
|
640
|
+
return { manifest, path };
|
|
641
|
+
}
|
|
642
|
+
async function readJsonIfPresent(path) {
|
|
643
|
+
try {
|
|
644
|
+
return JSON.parse(await readFile(path, "utf8"));
|
|
645
|
+
} catch (error) {
|
|
646
|
+
if (error.code === "ENOENT") return void 0;
|
|
647
|
+
throw error;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
async function listAssets(env) {
|
|
651
|
+
const root = assetsRoot(env);
|
|
652
|
+
const found = [];
|
|
653
|
+
for (const kind of ["datasets", "models"]) {
|
|
654
|
+
const repos = await readdir(join3(root, kind)).catch(() => []);
|
|
655
|
+
for (const repo of repos) {
|
|
656
|
+
for (const revision of await readdir(join3(root, kind, repo)).catch(() => [])) {
|
|
657
|
+
const path = join3(root, kind, repo, revision, "manifest.json");
|
|
658
|
+
const manifest = await readJsonIfPresent(path);
|
|
659
|
+
if (manifest) found.push({ ...manifest, manifest_path: path });
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
return found.sort((a, b) => b.fetched_at.localeCompare(a.fetched_at));
|
|
664
|
+
}
|
|
665
|
+
async function findAsset(reference, env) {
|
|
666
|
+
const [repo, revision] = reference.split("@");
|
|
667
|
+
const match = (await listAssets(env)).find((asset) => asset.repo === repo && (!revision || asset.revision.startsWith(revision.toLowerCase())));
|
|
668
|
+
if (!match) throw new Error(`No downloaded asset matches ${reference}. Run \`tt assets list\` to see downloads.`);
|
|
669
|
+
return match;
|
|
670
|
+
}
|
|
671
|
+
async function verifyAsset(manifest) {
|
|
672
|
+
const problems = [];
|
|
673
|
+
for (const [path, expected] of Object.entries(manifest.files)) {
|
|
674
|
+
const local = join3(manifest.snapshot_path, ...path.split("/"));
|
|
675
|
+
const target = await realpath(local).catch(() => null);
|
|
676
|
+
if (!target) {
|
|
677
|
+
problems.push(`${path}: missing from ${manifest.snapshot_path}`);
|
|
678
|
+
continue;
|
|
679
|
+
}
|
|
680
|
+
const algorithm = expected.startsWith("git-sha1:") ? "git-sha1" : "sha256";
|
|
681
|
+
if (await contentDigest(target, algorithm) !== expected) problems.push(`${path}: content does not match ${expected}`);
|
|
682
|
+
}
|
|
683
|
+
return { ok: problems.length === 0, problems };
|
|
684
|
+
}
|
|
685
|
+
function specLockPath(specPath) {
|
|
686
|
+
const name = basename(specPath).replace(/^\.(.+)\.tt-(?:local|agent)-[^/\\]*\.json$/, "$1");
|
|
687
|
+
return join3(dirname4(specPath), `${name.replace(/\.json$/i, "")}.lock.json`);
|
|
688
|
+
}
|
|
689
|
+
async function readSpecLock(specPath) {
|
|
690
|
+
const path = specLockPath(specPath);
|
|
691
|
+
const lock = await readJsonIfPresent(path);
|
|
692
|
+
if (lock && lock.version !== 1) throw new Error(`${path} has unsupported version ${String(lock.version)}.`);
|
|
693
|
+
return lock;
|
|
694
|
+
}
|
|
695
|
+
async function writeSpecLockEntry(specPath, key, entry) {
|
|
696
|
+
const lock = await readSpecLock(specPath) ?? { version: 1 };
|
|
697
|
+
await writeJsonAtomic(specLockPath(specPath), { ...lock, version: 1, [key]: sortedEntry(entry) });
|
|
698
|
+
}
|
|
699
|
+
function sortedEntry(entry) {
|
|
700
|
+
return {
|
|
701
|
+
repo: entry.repo,
|
|
702
|
+
revision: entry.revision,
|
|
703
|
+
files: Object.fromEntries(Object.entries(entry.files).sort(([a], [b]) => a.localeCompare(b)))
|
|
704
|
+
};
|
|
705
|
+
}
|
|
706
|
+
var PREFETCH_COMMAND = {
|
|
707
|
+
dataset: "tt datasets prefetch",
|
|
708
|
+
base_model: "tt models prefetch"
|
|
709
|
+
};
|
|
710
|
+
async function checkSpecLock(specPath, key, actual) {
|
|
711
|
+
const locked = (await readSpecLock(specPath))?.[key];
|
|
712
|
+
if (!locked) {
|
|
713
|
+
await writeSpecLockEntry(specPath, key, actual);
|
|
714
|
+
return "recorded";
|
|
715
|
+
}
|
|
716
|
+
const lockFile = basename(specLockPath(specPath));
|
|
717
|
+
const update = `If this change is intended, run \`${PREFETCH_COMMAND[key]}\` with this spec to update ${lockFile}.`;
|
|
718
|
+
const label = key === "dataset" ? "dataset" : "base model";
|
|
719
|
+
if (locked.repo !== actual.repo || locked.revision !== actual.revision) {
|
|
720
|
+
throw new SpecLockMismatchError(
|
|
721
|
+
`The spec's ${label} is ${actual.repo}@${actual.revision}, but ${lockFile} pins ${locked.repo}@${locked.revision}. ${update}`
|
|
722
|
+
);
|
|
723
|
+
}
|
|
724
|
+
if (key === "base_model") {
|
|
725
|
+
for (const path of Object.keys(locked.files)) {
|
|
726
|
+
if (actual.files[path] === void 0) {
|
|
727
|
+
throw new SpecLockMismatchError(`base model file ${path} is missing from the cached snapshot. ${update}`);
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
for (const [path, digest] of Object.entries(actual.files)) {
|
|
732
|
+
if (locked.files[path] === void 0) {
|
|
733
|
+
throw new SpecLockMismatchError(`${label} file ${path} is not in ${lockFile}. ${update}`);
|
|
734
|
+
}
|
|
735
|
+
if (locked.files[path] !== digest) {
|
|
736
|
+
throw new SpecLockMismatchError(
|
|
737
|
+
`${label} file ${path} is ${digest}, but ${lockFile} expects ${locked.files[path]}. The cached file differs from the one this project was locked to. ${update}`
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
return "matched";
|
|
742
|
+
}
|
|
743
|
+
function lockedRevision(lock, key, repo) {
|
|
744
|
+
const entry = lock?.[key];
|
|
745
|
+
return entry?.repo === repo && COMMIT_SHA.test(entry.revision) ? entry.revision : void 0;
|
|
746
|
+
}
|
|
747
|
+
function applySpecLock(request, lock) {
|
|
748
|
+
const source = request.dataset_prebuilt?.huggingface;
|
|
749
|
+
if (!source || source.revision) return request;
|
|
750
|
+
const revision = lockedRevision(lock, "dataset", source.repo);
|
|
751
|
+
if (!revision) return request;
|
|
752
|
+
return { ...request, dataset_prebuilt: { ...request.dataset_prebuilt, huggingface: { ...source, revision } } };
|
|
753
|
+
}
|
|
754
|
+
async function enforceSpecLock(args) {
|
|
755
|
+
if (args.config.dryRun) return;
|
|
756
|
+
const dataset = args.request.dataset_source;
|
|
757
|
+
if (dataset) {
|
|
758
|
+
const paths = Object.values(dataset.files).flatMap((file) => file ? [file.path] : []);
|
|
759
|
+
const { files: files2 } = await snapshotDigests(dataset.snapshot_path, paths);
|
|
760
|
+
await checkSpecLock(args.specPath, "dataset", { repo: dataset.repo, revision: dataset.revision, files: files2 });
|
|
761
|
+
}
|
|
762
|
+
if (args.config.paths.baseModel || args.config.gpu) return;
|
|
763
|
+
const model = resolveTrainingModel(args.request.spec_snapshot.base_model);
|
|
764
|
+
const revision = resolveRequestedBaseModelRevision(model.id, args.request.hyperparameters.base_model_revision);
|
|
765
|
+
if (!revision) return;
|
|
766
|
+
const hubCache = withHuggingFaceCacheEnvironment(process.env, args.config.paths.modelCache).HF_HUB_CACHE;
|
|
767
|
+
const snapshot = snapshotDirectory(hubCache, "model", model.id, revision);
|
|
768
|
+
if (!await stat(snapshot).then(() => true, () => false)) {
|
|
769
|
+
if ((await readSpecLock(args.specPath))?.base_model) {
|
|
770
|
+
throw new SpecLockMismatchError(
|
|
771
|
+
`Base model ${model.id}@${revision} is not downloaded (${snapshot}). Run \`tt models prefetch\` with this spec first.`
|
|
772
|
+
);
|
|
773
|
+
}
|
|
774
|
+
const { prefetchBaseModel: prefetchBaseModel2 } = await import("./prefetch-Q2ZA3SJ7.js");
|
|
775
|
+
await prefetchBaseModel2({ request: args.request, config: args.config, reporter: args.reporter });
|
|
776
|
+
}
|
|
777
|
+
const { files } = await snapshotDigests(snapshot);
|
|
778
|
+
await checkSpecLock(args.specPath, "base_model", { repo: model.id, revision, files });
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
// src/local-runtime/prefetch.ts
|
|
782
|
+
var NEMOTRON_BF16_FILE_DIGESTS = {
|
|
783
|
+
".gitattributes": { algorithm: "sha1", digest: "fe3f694ce14b577c9833fca6e83c5e26a7119127" },
|
|
784
|
+
"LICENSE": { algorithm: "sha1", digest: "4d76cc87c9edb280753e3bee3b499ae461113835" },
|
|
785
|
+
"README.md": { algorithm: "sha1", digest: "8ec43ee9f3ee345202e4a6ef752c51b2619f7425" },
|
|
786
|
+
"accuracy_plot.png": { algorithm: "sha256", digest: "1397995f8a5d34d819a069d86c02914990e9ecfc2fce101e233924da9cf5ddcf" },
|
|
787
|
+
"agentic_coding_benchmarks.png": { algorithm: "sha256", digest: "06cf5486ae94ecd7cc6f91ae3b4bae21c7435253a294cb89f54f7ec42eda8847" },
|
|
788
|
+
"bias.md": { algorithm: "sha1", digest: "cdcc8055f965222fabddabcabf22885d3a8bacc1" },
|
|
789
|
+
"chat_template.jinja": { algorithm: "sha1", digest: "d85b0c772f8fe585063847c5f6bf5ec48eb210be" },
|
|
790
|
+
"config.json": { algorithm: "sha1", digest: "993b612989e6aa67da45f908ea5983cb777e7678" },
|
|
791
|
+
"explainability.md": { algorithm: "sha1", digest: "2435f23c865e7daba6a6eceef2f214b86df75d24" },
|
|
792
|
+
"generation_config.json": { algorithm: "sha1", digest: "a41201df53f9d0769947989353b5d02e70491c56" },
|
|
793
|
+
"model.safetensors.index.json": { algorithm: "sha1", digest: "3cd409443b1896d9adc5c3981af301841f85329a" },
|
|
794
|
+
"special_tokens_map.json": { algorithm: "sha1", digest: "0451f37912edb7f8a4bf04d77eef13f6130f0515" },
|
|
795
|
+
"tokenizer.json": { algorithm: "sha256", digest: "623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7" },
|
|
796
|
+
"tokenizer_config.json": { algorithm: "sha1", digest: "c96e5ad00986bfa0b666d4bc425666f1c51d6d1b" },
|
|
797
|
+
"model-00001-of-00014.safetensors": { algorithm: "sha256", digest: "7a3e74d24969eac6657cb9b6422091fd062b7d30fd4bc90ecee6c3d7f692626a" },
|
|
798
|
+
"model-00002-of-00014.safetensors": { algorithm: "sha256", digest: "a80c3846e959f6dda34aa031cbb668b766643691ac7e5c9f43f62e6fb1221059" },
|
|
799
|
+
"model-00003-of-00014.safetensors": { algorithm: "sha256", digest: "996bf3a46668e14075cb82259390267c2b1f71446caeb3a5f1f9e0bd7a079ae7" },
|
|
800
|
+
"model-00004-of-00014.safetensors": { algorithm: "sha256", digest: "cce1d79e7ff2e2b746b9d97166156aa1dcec796116675cc7fd1aa70a8d73ccb2" },
|
|
801
|
+
"model-00005-of-00014.safetensors": { algorithm: "sha256", digest: "9338f59d3fd5f074288c71a9559452d5b5c581da00e9ccb9fe7aa2ce9b04d2ad" },
|
|
802
|
+
"model-00006-of-00014.safetensors": { algorithm: "sha256", digest: "e947fa4097c68263ba3d097fceb32a300a22cebd9d909fb5198af7d18a4022fe" },
|
|
803
|
+
"model-00007-of-00014.safetensors": { algorithm: "sha256", digest: "c7362630bf4256275ca58354ed8f31506cec67ffab30573e753703fd5ae48795" },
|
|
804
|
+
"model-00008-of-00014.safetensors": { algorithm: "sha256", digest: "1b33a5a6b5505b25b57c90ddda19e9cd1598176468917df0c56116fa6f6072cb" },
|
|
805
|
+
"model-00009-of-00014.safetensors": { algorithm: "sha256", digest: "fcc9b48caff427beaf1f87b78cf4ab42bc2a3c43b3c487bccb013ba22d7468b2" },
|
|
806
|
+
"model-00010-of-00014.safetensors": { algorithm: "sha256", digest: "d1924fd54d1b77a15aff8df591a317e8a82a58e46d6f5742fa21e90e0be3cfa6" },
|
|
807
|
+
"model-00011-of-00014.safetensors": { algorithm: "sha256", digest: "e9d0b83bcf04bd4d5758b94e6495a3f1595d121d82faffaa27ce6a884e2c8757" },
|
|
808
|
+
"model-00012-of-00014.safetensors": { algorithm: "sha256", digest: "fce184f112cb0b97024cadc60f993f6f660e365d04e21c240fb3c75747bd3a01" },
|
|
809
|
+
"model-00013-of-00014.safetensors": { algorithm: "sha256", digest: "f427d037d2a3e4d44323325aa38571e3551e8ad702f469c3dacc4e59ea2e50d1" },
|
|
810
|
+
"model-00014-of-00014.safetensors": { algorithm: "sha256", digest: "64577b275ca4e7e5266eae0903674f7f46ec2a8cbf4f4f1a3207f80d503cd1d0" },
|
|
811
|
+
"privacy.md": { algorithm: "sha1", digest: "e3bf30aa1c42a488298ef37383647dc99e1cd39e" },
|
|
812
|
+
"safety.md": { algorithm: "sha1", digest: "f53bf94a4635862eed5cf3e0bc799b93f81d20ac" }
|
|
813
|
+
};
|
|
814
|
+
var MUSE_GLIMMER_FILE_DIGESTS = {
|
|
815
|
+
".gitattributes": { algorithm: "sha1", digest: "52373fe24473b1aa44333d318f578ae6bf04b49b" },
|
|
816
|
+
"LICENSE": { algorithm: "sha1", digest: "d645695673349e3947e8e5ae42332d0ac3164cd7" },
|
|
817
|
+
"README.md": { algorithm: "sha1", digest: "a25f87c1c1fbd88f367a552fd14427baa8cb947e" },
|
|
818
|
+
"USAGE_POLICY.md": { algorithm: "sha1", digest: "1a9ed6cffc54585daf9cc904c07cb6e436a9444b" },
|
|
819
|
+
"chat_template.jinja": { algorithm: "sha1", digest: "7507f3c9f38809152732c045df3977848f3916a6" },
|
|
820
|
+
"config.json": { algorithm: "sha1", digest: "190826dc834c13c86b8dc68e775a888a852d7c34" },
|
|
821
|
+
"generation_config.json": { algorithm: "sha1", digest: "b69a50a4239f42707d54a479e0143b8ee56c8bcc" },
|
|
822
|
+
"model.safetensors.index.json": { algorithm: "sha1", digest: "00b257ef0c8bce1b001a6266f6b1f66f4fe16a6b" },
|
|
823
|
+
"processor_config.json": { algorithm: "sha1", digest: "ec9a07be13ea1dadf7395131871753b13f406c26" },
|
|
824
|
+
"tokenizer_config.json": { algorithm: "sha1", digest: "fe7f0bb90fb0d0288e3a505974ae0e4baf1d3e61" },
|
|
825
|
+
"tokenizer.json": { algorithm: "sha256", digest: "c9dbee66967b58f31a7c27f723c3760da3526ccd0427578e8905b0abb0031c4d" },
|
|
826
|
+
"model-00001-of-00002.safetensors": { algorithm: "sha256", digest: "8eef61530e1283642c77ce2e6721feb5c6f348fa055c00e90f2844a136372694" },
|
|
827
|
+
"model-00002-of-00002.safetensors": { algorithm: "sha256", digest: "b58cc2144ba1ba1af4420f67f4ca3ced7f09298510b80464cc75018a0be14381" }
|
|
828
|
+
};
|
|
829
|
+
function certifiedFileDigests(modelId) {
|
|
830
|
+
switch (modelId) {
|
|
831
|
+
case "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16":
|
|
832
|
+
return NEMOTRON_BF16_FILE_DIGESTS;
|
|
833
|
+
case "meta-models/Muse-Glimmer-30B":
|
|
834
|
+
return MUSE_GLIMMER_FILE_DIGESTS;
|
|
835
|
+
default:
|
|
836
|
+
return void 0;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
async function certifiedFileDigest(path, expected) {
|
|
840
|
+
const metadata = await stat2(path);
|
|
841
|
+
const digest = createHash3(expected.algorithm);
|
|
842
|
+
if (expected.algorithm === "sha1") digest.update(`blob ${metadata.size}\0`);
|
|
843
|
+
for await (const chunk of createReadStream2(path, { highWaterMark: 8 * 1024 * 1024 })) digest.update(chunk);
|
|
844
|
+
return digest.digest("hex");
|
|
845
|
+
}
|
|
846
|
+
function isStrictPhysicalDescendant(parent, child) {
|
|
847
|
+
const candidate = relative2(parent, child);
|
|
848
|
+
return candidate !== "" && candidate !== ".." && !candidate.startsWith(`..${sep2}`) && !isAbsolute(candidate);
|
|
849
|
+
}
|
|
850
|
+
async function verifySnapshotCacheLocation(path, modelId, revision, modelCache) {
|
|
851
|
+
const repository = `models--${modelId.replaceAll("/", "--")}`;
|
|
852
|
+
const layout = resolveHuggingFaceCacheLayout(modelCache);
|
|
853
|
+
const repositoryRoot = join4(layout.hubCache, repository);
|
|
854
|
+
const expectedPath = join4(repositoryRoot, "snapshots", revision);
|
|
855
|
+
const [
|
|
856
|
+
physicalCacheRoot,
|
|
857
|
+
physicalHubRoot,
|
|
858
|
+
physicalRepositoryRoot,
|
|
859
|
+
physicalBlobRoot,
|
|
860
|
+
physicalPath,
|
|
861
|
+
physicalExpectedPath
|
|
862
|
+
] = await Promise.all([
|
|
863
|
+
realpath2(layout.hfHome).catch(() => null),
|
|
864
|
+
realpath2(layout.hubCache).catch(() => null),
|
|
865
|
+
realpath2(repositoryRoot).catch(() => null),
|
|
866
|
+
realpath2(join4(repositoryRoot, "blobs")).catch(() => null),
|
|
867
|
+
realpath2(path).catch(() => null),
|
|
868
|
+
realpath2(expectedPath).catch(() => null)
|
|
869
|
+
]);
|
|
870
|
+
if (!physicalCacheRoot || !physicalHubRoot || !physicalRepositoryRoot || !physicalBlobRoot || !physicalPath || !physicalExpectedPath || physicalPath !== physicalExpectedPath) {
|
|
871
|
+
throw new Error(
|
|
872
|
+
`${modelId} must use the certified Hugging Face cache snapshot ${expectedPath}; got ${path}.`
|
|
873
|
+
);
|
|
874
|
+
}
|
|
875
|
+
if (!isStrictPhysicalDescendant(physicalCacheRoot, physicalHubRoot)) {
|
|
876
|
+
throw new Error(`Hugging Face hub cache escapes its configured cache root: ${layout.hubCache}`);
|
|
877
|
+
}
|
|
878
|
+
if (!isStrictPhysicalDescendant(physicalHubRoot, physicalRepositoryRoot)) {
|
|
879
|
+
throw new Error(`Hugging Face repository escapes its configured hub cache: ${repositoryRoot}`);
|
|
880
|
+
}
|
|
881
|
+
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalBlobRoot)) {
|
|
882
|
+
throw new Error(`Hugging Face blob store escapes its repository: ${join4(repositoryRoot, "blobs")}`);
|
|
883
|
+
}
|
|
884
|
+
if (!isStrictPhysicalDescendant(physicalRepositoryRoot, physicalExpectedPath)) {
|
|
885
|
+
throw new Error(`Hugging Face snapshot escapes its configured repository: ${expectedPath}`);
|
|
886
|
+
}
|
|
887
|
+
return { repositoryRoot, physicalBlobRoot };
|
|
888
|
+
}
|
|
889
|
+
async function requireReportedCachePath(label, reported, expected) {
|
|
890
|
+
const [physicalReported, physicalExpected] = await Promise.all([
|
|
891
|
+
reported ? realpath2(reported).catch(() => null) : null,
|
|
892
|
+
realpath2(expected).catch(() => null)
|
|
893
|
+
]);
|
|
894
|
+
if (!physicalReported || !physicalExpected || physicalReported !== physicalExpected) {
|
|
895
|
+
throw new Error(`Model prefetch returned an unexpected ${label}: ${String(reported)}.`);
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
async function verifyModelPrefetchCacheReport(args) {
|
|
899
|
+
if (!args.output.snapshot_path) {
|
|
900
|
+
throw new Error("Model prefetch did not return a snapshot path.");
|
|
901
|
+
}
|
|
902
|
+
const effectiveHubCache = resolveHuggingFaceCacheLayout(args.modelCache).hubCache;
|
|
903
|
+
await Promise.all([
|
|
904
|
+
requireReportedCachePath("model cache", args.output.model_cache, args.modelCache),
|
|
905
|
+
requireReportedCachePath("HF home", args.output.hf_home, args.modelCache),
|
|
906
|
+
requireReportedCachePath("hub cache", args.output.hub_cache, effectiveHubCache)
|
|
907
|
+
]);
|
|
908
|
+
await verifySnapshotCacheLocation(
|
|
909
|
+
args.output.snapshot_path,
|
|
910
|
+
args.modelId,
|
|
911
|
+
args.revision,
|
|
912
|
+
args.modelCache
|
|
913
|
+
);
|
|
914
|
+
}
|
|
915
|
+
async function verifyCertifiedLocalSnapshot(path, modelId, snapshotFiles, modelCache) {
|
|
916
|
+
const revision = defaultBaseModelRevision(modelId);
|
|
917
|
+
if (!revision) return;
|
|
918
|
+
const digests = certifiedFileDigests(modelId);
|
|
919
|
+
if (!digests) return;
|
|
920
|
+
if (!modelCache) {
|
|
921
|
+
throw new Error(`A certified local snapshot for ${modelId} requires paths.modelCache.`);
|
|
922
|
+
}
|
|
923
|
+
const { physicalBlobRoot } = await verifySnapshotCacheLocation(
|
|
924
|
+
path,
|
|
925
|
+
modelId,
|
|
926
|
+
revision,
|
|
927
|
+
modelCache
|
|
928
|
+
);
|
|
929
|
+
const physicalFiles = /* @__PURE__ */ new Map();
|
|
930
|
+
for (const file of snapshotFiles) {
|
|
931
|
+
const metadata = await lstat(file);
|
|
932
|
+
const target = await realpath2(file);
|
|
933
|
+
if (metadata.isSymbolicLink()) {
|
|
934
|
+
if (!isStrictPhysicalDescendant(physicalBlobRoot, target)) {
|
|
935
|
+
throw new Error(`Certified snapshot symbolic link escapes its Hugging Face blob store: ${file}`);
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
physicalFiles.set(file, target);
|
|
939
|
+
}
|
|
940
|
+
const actualFiles = new Set(snapshotFiles.map((file) => relative2(path, file).split("\\").join("/")));
|
|
941
|
+
const expectedFiles = new Set(Object.keys(digests));
|
|
942
|
+
const unexpected = [...actualFiles].filter((name) => !expectedFiles.has(name)).sort();
|
|
943
|
+
if (unexpected.length > 0) {
|
|
944
|
+
throw new Error(`Certified snapshot contains unexpected file: ${join4(path, unexpected[0])}`);
|
|
945
|
+
}
|
|
946
|
+
for (const [name, expected] of Object.entries(digests)) {
|
|
947
|
+
if (!actualFiles.has(name)) {
|
|
948
|
+
throw new Error(`Certified snapshot file is missing: ${join4(path, name)}`);
|
|
949
|
+
}
|
|
950
|
+
const file = join4(path, name);
|
|
951
|
+
const physicalFile = physicalFiles.get(file);
|
|
952
|
+
if (await certifiedFileDigest(physicalFile, expected) !== expected.digest) {
|
|
953
|
+
throw new Error(`Certified snapshot file checksum mismatch: ${file}`);
|
|
954
|
+
}
|
|
955
|
+
if (await realpath2(file) !== physicalFile) {
|
|
956
|
+
throw new Error(`Certified snapshot file changed during verification: ${file}`);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
function buildModelPrefetchPayload(request, config) {
|
|
961
|
+
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
962
|
+
const revision = resolveRequestedBaseModelRevision(
|
|
963
|
+
model.id,
|
|
964
|
+
request.hyperparameters.base_model_revision
|
|
965
|
+
);
|
|
966
|
+
return {
|
|
967
|
+
base_model: model.id,
|
|
968
|
+
...revision ? { revision } : {},
|
|
969
|
+
...config.paths.modelCache ? { model_cache: resolve4(config.paths.modelCache) } : {}
|
|
970
|
+
};
|
|
971
|
+
}
|
|
972
|
+
function buildModelSourcePrefetchPayload(source, config) {
|
|
973
|
+
const model = resolveTrainingModel(source.repo);
|
|
974
|
+
const revision = source.revision || isCertifiedTrainingModel(model.id) ? resolveRequestedBaseModelRevision(model.id, source.revision) : void 0;
|
|
975
|
+
return {
|
|
976
|
+
base_model: model.id,
|
|
977
|
+
...revision ? { revision } : { resolve_latest: true },
|
|
978
|
+
...config.paths.modelCache ? { model_cache: resolve4(config.paths.modelCache) } : {}
|
|
979
|
+
};
|
|
980
|
+
}
|
|
981
|
+
function safeName(value) {
|
|
982
|
+
return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "model";
|
|
983
|
+
}
|
|
984
|
+
function transformersWeightName(name) {
|
|
985
|
+
const lower = name.split(/[\\/]/).at(-1)?.toLowerCase() ?? "";
|
|
986
|
+
return lower.endsWith(".safetensors") && !lower.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(lower);
|
|
987
|
+
}
|
|
988
|
+
async function verifyLocalBaseModel(path, expectedModelId, modelCache) {
|
|
989
|
+
const root = await lstat(path).catch(() => null);
|
|
990
|
+
if (!root) throw new Error(`paths.baseModel is set to ${path}, but that path does not exist.`);
|
|
991
|
+
if (root.isSymbolicLink()) {
|
|
992
|
+
throw new Error(`paths.baseModel must not itself be a symbolic link: ${path}`);
|
|
993
|
+
}
|
|
994
|
+
if (!root.isDirectory()) {
|
|
995
|
+
throw new Error(
|
|
996
|
+
`paths.baseModel must be a Hugging Face snapshot directory, not a standalone file or archive: ${path}`
|
|
997
|
+
);
|
|
998
|
+
}
|
|
999
|
+
const files = [];
|
|
1000
|
+
const visit = async (directory) => {
|
|
1001
|
+
for (const entry of await readdir2(directory, { withFileTypes: true })) {
|
|
1002
|
+
const child = join4(directory, entry.name);
|
|
1003
|
+
if (entry.isSymbolicLink()) {
|
|
1004
|
+
const metadata2 = await stat2(child);
|
|
1005
|
+
if (!metadata2.isFile()) {
|
|
1006
|
+
throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
|
|
1007
|
+
}
|
|
1008
|
+
files.push({ path: child, size: metadata2.size });
|
|
1009
|
+
continue;
|
|
1010
|
+
}
|
|
1011
|
+
if (entry.isDirectory()) {
|
|
1012
|
+
await visit(child);
|
|
1013
|
+
continue;
|
|
1014
|
+
}
|
|
1015
|
+
const metadata = await stat2(child);
|
|
1016
|
+
if (metadata.isFile()) files.push({ path: child, size: metadata.size });
|
|
1017
|
+
}
|
|
1018
|
+
};
|
|
1019
|
+
await visit(path);
|
|
1020
|
+
const nonEmpty = files.filter((file) => file.size > 0);
|
|
1021
|
+
const weights = nonEmpty.filter((file) => transformersWeightName(file.path));
|
|
1022
|
+
if (weights.length === 0) {
|
|
1023
|
+
throw new Error(`Local base-model directory contains no non-empty Transformers model weights: ${path}`);
|
|
1024
|
+
}
|
|
1025
|
+
const required = async (name) => {
|
|
1026
|
+
const metadata = await stat2(join4(path, name)).catch(() => null);
|
|
1027
|
+
return Boolean(metadata?.isFile() && metadata.size > 0);
|
|
1028
|
+
};
|
|
1029
|
+
if (!await required("config.json")) throw new Error(`Local base-model directory is missing config.json: ${path}`);
|
|
1030
|
+
try {
|
|
1031
|
+
const config = JSON.parse(await readFile2(join4(path, "config.json"), "utf8"));
|
|
1032
|
+
assertCertifiedBaseModelConfig(
|
|
1033
|
+
config,
|
|
1034
|
+
`Local base-model config ${join4(path, "config.json")}`,
|
|
1035
|
+
expectedModelId
|
|
1036
|
+
);
|
|
1037
|
+
} catch (error) {
|
|
1038
|
+
const detail = error instanceof Error ? `: ${error.message}` : "";
|
|
1039
|
+
throw new Error(
|
|
1040
|
+
`Local base-model directory has an invalid or unsupported config.json: ${path}${detail}`,
|
|
1041
|
+
{ cause: error }
|
|
1042
|
+
);
|
|
1043
|
+
}
|
|
1044
|
+
if (expectedModelId) {
|
|
1045
|
+
await verifyCertifiedLocalSnapshot(
|
|
1046
|
+
path,
|
|
1047
|
+
expectedModelId,
|
|
1048
|
+
files.map((file) => file.path),
|
|
1049
|
+
modelCache
|
|
1050
|
+
);
|
|
1051
|
+
}
|
|
1052
|
+
const vocabNames = [
|
|
1053
|
+
"tokenizer.json",
|
|
1054
|
+
"tokenizer.model",
|
|
1055
|
+
"sentencepiece.bpe.model",
|
|
1056
|
+
"spiece.model",
|
|
1057
|
+
"vocab.json",
|
|
1058
|
+
"tokenizer.tiktoken"
|
|
1059
|
+
];
|
|
1060
|
+
if (!await required("tokenizer_config.json") || !(await Promise.all(vocabNames.map(required))).some(Boolean)) {
|
|
1061
|
+
throw new Error(`Local base-model directory is missing tokenizer metadata or vocabulary: ${path}`);
|
|
1062
|
+
}
|
|
1063
|
+
const inventory = new Set(nonEmpty.map((file) => resolve4(file.path)));
|
|
1064
|
+
for (const file of files.filter((candidate) => candidate.path.endsWith(".index.json"))) {
|
|
1065
|
+
const parsed = JSON.parse(await readFile2(file.path, "utf8"));
|
|
1066
|
+
if (!parsed.weight_map || typeof parsed.weight_map !== "object" || Array.isArray(parsed.weight_map)) {
|
|
1067
|
+
throw new Error(`Local base-model weight index is invalid: ${file.path}`);
|
|
1068
|
+
}
|
|
1069
|
+
for (const shard of new Set(Object.values(parsed.weight_map))) {
|
|
1070
|
+
if (typeof shard !== "string" || !shard || /^(?:[A-Za-z]:|[/\\])/.test(shard) || shard.split(/[/\\]/).includes("..")) {
|
|
1071
|
+
throw new Error("Local base-model weight index has an invalid indexed weight shard path");
|
|
1072
|
+
}
|
|
1073
|
+
if (!inventory.has(resolve4(path, shard)) || !await required(shard)) {
|
|
1074
|
+
throw new Error(`Local base-model directory is missing indexed weight shard: ${String(shard)}`);
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
return {
|
|
1079
|
+
fileCount: files.length,
|
|
1080
|
+
sizeBytes: files.reduce((total, file) => total + file.size, 0)
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
1083
|
+
async function prefetchBaseModel(args) {
|
|
1084
|
+
if (!args.request === !args.source) throw new Error("Model prefetch needs either a spec request or a source repo.");
|
|
1085
|
+
const payload = {
|
|
1086
|
+
...args.source ? buildModelSourcePrefetchPayload(args.source, args.config) : buildModelPrefetchPayload(args.request, args.config),
|
|
1087
|
+
...args.localOnly ? { local_files_only: true } : {}
|
|
1088
|
+
};
|
|
1089
|
+
const artifactDir = resolve4(
|
|
1090
|
+
args.config.artifactRoot,
|
|
1091
|
+
args.localOnly ? "verify-base" : "prefetch",
|
|
1092
|
+
`${safeName(payload.base_model)}-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
|
|
1093
|
+
);
|
|
1094
|
+
await mkdir2(artifactDir, { recursive: true });
|
|
1095
|
+
if (args.request && args.config.paths.baseModel) {
|
|
1096
|
+
const localPath = resolve4(args.config.paths.baseModel);
|
|
1097
|
+
const modelId = args.request.spec_snapshot.base_model;
|
|
1098
|
+
const verified = await withProgressHeartbeat(args.reporter, {
|
|
1099
|
+
stage: "model_prefetch",
|
|
1100
|
+
status: "running",
|
|
1101
|
+
message: "Verifying local model checksums. Large cached models can take several minutes."
|
|
1102
|
+
}, () => verifyLocalBaseModel(
|
|
1103
|
+
localPath,
|
|
1104
|
+
modelId,
|
|
1105
|
+
args.config.paths.modelCache
|
|
1106
|
+
));
|
|
1107
|
+
return {
|
|
1108
|
+
ok: true,
|
|
1109
|
+
status: args.localOnly ? "completed" : "skipped",
|
|
1110
|
+
base_model: args.request.spec_snapshot.base_model,
|
|
1111
|
+
local_base_model_path: localPath,
|
|
1112
|
+
file_count: verified.fileCount,
|
|
1113
|
+
size_bytes: verified.sizeBytes,
|
|
1114
|
+
artifact_dir: artifactDir,
|
|
1115
|
+
reason: args.localOnly ? "Verified the configured local base-model artifact; no network access was used." : "paths.baseModel points at a verified local base-model artifact; no Hugging Face download is needed."
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
const inputPath = join4(artifactDir, "prefetch-input.json");
|
|
1119
|
+
const outputPath = join4(artifactDir, "prefetch-output.json");
|
|
1120
|
+
const logPath = join4(artifactDir, "prefetch.log");
|
|
1121
|
+
await writeJson(inputPath, payload);
|
|
1122
|
+
const entrypoint = buildBundledPythonCommand(
|
|
1123
|
+
"prefetch.py",
|
|
1124
|
+
["--input", inputPath, "--output", outputPath]
|
|
1125
|
+
);
|
|
1126
|
+
const downloadEvent = {
|
|
1127
|
+
stage: "model_prefetch",
|
|
1128
|
+
status: "running",
|
|
1129
|
+
message: args.localOnly ? "Verifying cached model files. Large models can take several minutes." : "Downloading and verifying model files. Cached files still need checksum verification; large models can take several minutes.",
|
|
1130
|
+
details: {
|
|
1131
|
+
base_model: payload.base_model,
|
|
1132
|
+
model_cache: payload.model_cache ?? null,
|
|
1133
|
+
command: entrypoint.displayCommand,
|
|
1134
|
+
log_path: logPath
|
|
1135
|
+
}
|
|
1136
|
+
};
|
|
1137
|
+
const cacheEnvironment = withHuggingFaceCacheEnvironment(
|
|
1138
|
+
minimalMachineLearningEnvironment(process.env),
|
|
1139
|
+
payload.model_cache
|
|
1140
|
+
);
|
|
1141
|
+
const effectiveModelCache = cacheEnvironment.HF_HOME;
|
|
1142
|
+
const effectiveHubCache = resolveHuggingFaceCacheLayout(effectiveModelCache).hubCache;
|
|
1143
|
+
const env = withBundledPythonEnvironment(cacheEnvironment);
|
|
1144
|
+
const { exitCode } = await withProgressHeartbeat(args.reporter, downloadEvent, () => runLoggedProcess({
|
|
1145
|
+
command: entrypoint.command,
|
|
1146
|
+
commandArgs: entrypoint.commandArgs,
|
|
1147
|
+
env,
|
|
1148
|
+
logPath,
|
|
1149
|
+
// A base model can be several gigabytes. Surface the downloader's progress
|
|
1150
|
+
// by default; callers can still opt out by omitting the reporter (`--quiet`).
|
|
1151
|
+
reporter: args.reporter ? { ...args.reporter, verbose: true } : void 0,
|
|
1152
|
+
stage: "model_prefetch"
|
|
1153
|
+
}));
|
|
1154
|
+
if (exitCode !== 0) {
|
|
1155
|
+
throw new Error(`Model prefetch exited with code ${exitCode}. See ${logPath}.`);
|
|
1156
|
+
}
|
|
1157
|
+
const output = JSON.parse(await readFile2(outputPath, "utf8"));
|
|
1158
|
+
const snapshotRevision = baseModelRevisionSchema.safeParse(output.snapshot_revision);
|
|
1159
|
+
if (!snapshotRevision.success) {
|
|
1160
|
+
throw new Error(
|
|
1161
|
+
"Model prefetch did not resolve the base model to a 40-character immutable commit SHA."
|
|
1162
|
+
);
|
|
1163
|
+
}
|
|
1164
|
+
if (output.base_model !== payload.base_model) {
|
|
1165
|
+
throw new Error(`Model prefetch returned ${String(output.base_model)} for requested model ${payload.base_model}.`);
|
|
1166
|
+
}
|
|
1167
|
+
if (payload.revision && snapshotRevision.data !== payload.revision.toLowerCase()) {
|
|
1168
|
+
throw new Error(
|
|
1169
|
+
`Model prefetch resolved a different revision (${snapshotRevision.data}) than requested (${payload.revision}).`
|
|
1170
|
+
);
|
|
1171
|
+
}
|
|
1172
|
+
await verifyModelPrefetchCacheReport({
|
|
1173
|
+
modelId: payload.base_model,
|
|
1174
|
+
revision: snapshotRevision.data,
|
|
1175
|
+
modelCache: effectiveModelCache,
|
|
1176
|
+
output
|
|
1177
|
+
});
|
|
1178
|
+
if (defaultBaseModelRevision(payload.base_model)) {
|
|
1179
|
+
await withProgressHeartbeat(args.reporter, {
|
|
1180
|
+
stage: "model_prefetch",
|
|
1181
|
+
status: "running",
|
|
1182
|
+
message: "Checking model files against the certified revision's checksums."
|
|
1183
|
+
}, () => verifyLocalBaseModel(
|
|
1184
|
+
output.snapshot_path,
|
|
1185
|
+
payload.base_model,
|
|
1186
|
+
effectiveModelCache
|
|
1187
|
+
));
|
|
1188
|
+
}
|
|
1189
|
+
const recorded = args.localOnly ? void 0 : await withProgressHeartbeat(args.reporter, {
|
|
1190
|
+
stage: "model_prefetch",
|
|
1191
|
+
status: "running",
|
|
1192
|
+
message: "Verifying model files and recording the local asset manifest."
|
|
1193
|
+
}, () => recordAssetDownload({
|
|
1194
|
+
kind: "model",
|
|
1195
|
+
repo: payload.base_model,
|
|
1196
|
+
revision: snapshotRevision.data,
|
|
1197
|
+
snapshotPath: output.snapshot_path
|
|
1198
|
+
}));
|
|
1199
|
+
await args.reporter?.onEvent?.({
|
|
1200
|
+
stage: "model_prefetch",
|
|
1201
|
+
status: "completed",
|
|
1202
|
+
message: "Base model is available in the local Hugging Face cache.",
|
|
1203
|
+
details: {
|
|
1204
|
+
base_model: output.base_model ?? payload.base_model,
|
|
1205
|
+
hf_home: effectiveModelCache,
|
|
1206
|
+
hub_cache: effectiveHubCache,
|
|
1207
|
+
snapshot_path: output.snapshot_path ?? null,
|
|
1208
|
+
snapshot_revision: snapshotRevision.data,
|
|
1209
|
+
file_count: output.file_count ?? null,
|
|
1210
|
+
size_bytes: output.size_bytes ?? null,
|
|
1211
|
+
verified_blob_count: output.verified_blob_count ?? null
|
|
1212
|
+
}
|
|
1213
|
+
});
|
|
1214
|
+
return {
|
|
1215
|
+
ok: true,
|
|
1216
|
+
status: "completed",
|
|
1217
|
+
base_model: output.base_model ?? payload.base_model,
|
|
1218
|
+
model_cache: effectiveModelCache,
|
|
1219
|
+
hf_home: effectiveModelCache,
|
|
1220
|
+
hub_cache: effectiveHubCache,
|
|
1221
|
+
snapshot_path: output.snapshot_path,
|
|
1222
|
+
snapshot_revision: snapshotRevision.data,
|
|
1223
|
+
file_count: output.file_count,
|
|
1224
|
+
size_bytes: output.size_bytes,
|
|
1225
|
+
verified_blob_count: output.verified_blob_count,
|
|
1226
|
+
artifact_dir: artifactDir,
|
|
1227
|
+
input_path: inputPath,
|
|
1228
|
+
output_path: outputPath,
|
|
1229
|
+
log_uri: fileUri(logPath),
|
|
1230
|
+
command: entrypoint.displayCommand,
|
|
1231
|
+
...recorded ? { manifest_path: recorded.path } : {}
|
|
1232
|
+
};
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
export {
|
|
1236
|
+
minimalMachineLearningEnvironment,
|
|
1237
|
+
withHuggingFaceCacheEnvironment,
|
|
1238
|
+
withOfflineHuggingFaceCacheEnvironment,
|
|
1239
|
+
localRuntimePackageRoot,
|
|
1240
|
+
reportInBackground,
|
|
1241
|
+
withProgressHeartbeat,
|
|
1242
|
+
sanitizeLogLine,
|
|
1243
|
+
forwardStreamLines,
|
|
1244
|
+
createProgressRenderer,
|
|
1245
|
+
progressTty,
|
|
1246
|
+
progressColumns,
|
|
1247
|
+
buildDecisionPythonCommand,
|
|
1248
|
+
withDecisionPythonEnvironment,
|
|
1249
|
+
buildBundledPythonCommand,
|
|
1250
|
+
withBundledPythonEnvironment,
|
|
1251
|
+
buildFoundationPythonCommand,
|
|
1252
|
+
withFoundationPythonEnvironment,
|
|
1253
|
+
ProcessCancelledError,
|
|
1254
|
+
runLoggedProcess,
|
|
1255
|
+
snapshotDigests,
|
|
1256
|
+
recordAssetDownload,
|
|
1257
|
+
listAssets,
|
|
1258
|
+
findAsset,
|
|
1259
|
+
verifyAsset,
|
|
1260
|
+
specLockPath,
|
|
1261
|
+
readSpecLock,
|
|
1262
|
+
writeSpecLockEntry,
|
|
1263
|
+
applySpecLock,
|
|
1264
|
+
enforceSpecLock,
|
|
1265
|
+
verifyModelPrefetchCacheReport,
|
|
1266
|
+
buildModelPrefetchPayload,
|
|
1267
|
+
buildModelSourcePrefetchPayload,
|
|
1268
|
+
safeName,
|
|
1269
|
+
verifyLocalBaseModel,
|
|
1270
|
+
prefetchBaseModel
|
|
1271
|
+
};
|
|
1272
|
+
//# sourceMappingURL=chunk-ST6ZSEUH.js.map
|