@tuned-tensor/cli 0.9.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/dist/chunk-DAP2Z7EZ.js +4400 -0
- package/dist/chunk-DAP2Z7EZ.js.map +1 -0
- package/dist/index.js +324 -70
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.js +1630 -0
- package/dist/local-runtime.js.map +1 -0
- package/docs/local-runtime/architecture.md +91 -0
- package/docs/local-runtime/spark.md +107 -0
- package/examples/local-runtime/dry-runner.json +14 -0
- package/examples/local-runtime/general-regression.jsonl +4 -0
- package/examples/local-runtime/local-runner.json +25 -0
- package/examples/local-runtime/smoke-spec.json +41 -0
- package/package.json +15 -5
- package/training/local-runner/pyproject.toml +25 -0
- package/training/local-runner/src/evaluate.py +274 -0
- package/training/local-runner/src/model_contract.py +47 -0
- package/training/local-runner/src/prefetch.py +192 -0
- package/training/local-runner/src/serve.py +272 -0
- package/training/local-runner/src/sft_data.py +97 -0
- package/training/local-runner/src/train.py +289 -0
- package/training/local-runner/uv.lock +1204 -0
|
@@ -0,0 +1,4400 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/local-runtime/artifacts.ts
|
|
4
|
+
import { createHash, randomUUID } from "crypto";
|
|
5
|
+
import { createReadStream } from "fs";
|
|
6
|
+
import { copyFile, lstat, mkdir, open, readFile, readdir, realpath, rename, stat, writeFile } from "fs/promises";
|
|
7
|
+
import { dirname, isAbsolute, join, relative, resolve, sep } from "path";
|
|
8
|
+
import { createGunzip } from "zlib";
|
|
9
|
+
var RUN_OWNER_FILE = ".tt-local-run-owner.json";
|
|
10
|
+
var ARTIFACT_WORKFLOW_LOCK_FILE = ".tt-local-workflow.lock";
|
|
11
|
+
function fileUri(path) {
|
|
12
|
+
return `file://${resolve(path)}`;
|
|
13
|
+
}
|
|
14
|
+
function defaultArtifactPrefix(input) {
|
|
15
|
+
return join("users", input.userId, "specs", input.behaviorSpecId, "runs", input.runId);
|
|
16
|
+
}
|
|
17
|
+
function resolveRunArtifacts(args) {
|
|
18
|
+
const root = resolve(args.artifactRoot);
|
|
19
|
+
const safePrefix = args.prefix.replace(/^[/\\]+/, "");
|
|
20
|
+
const runDir = resolve(root, safePrefix);
|
|
21
|
+
const containment = relative(root, runDir);
|
|
22
|
+
if (containment === ".." || containment.startsWith(`..${sep}`) || isAbsolute(containment)) {
|
|
23
|
+
throw new Error(`Artifact prefix escapes artifactRoot: ${args.prefix}`);
|
|
24
|
+
}
|
|
25
|
+
const trainingDir = join(runDir, "training");
|
|
26
|
+
return {
|
|
27
|
+
root,
|
|
28
|
+
prefix: safePrefix,
|
|
29
|
+
runDir,
|
|
30
|
+
trainingJsonl: join(runDir, "training.jsonl"),
|
|
31
|
+
stageMetadataJson: join(runDir, "stage-metadata.json"),
|
|
32
|
+
trainingReportJson: join(runDir, "training-report.json"),
|
|
33
|
+
baselineEvalJson: join(runDir, "baseline-eval.json"),
|
|
34
|
+
candidateEvalJson: join(runDir, "candidate-eval.json"),
|
|
35
|
+
generalBaselineEvalJson: join(runDir, "general-baseline-eval.json"),
|
|
36
|
+
generalCandidateEvalJson: join(runDir, "general-candidate-eval.json"),
|
|
37
|
+
runReportJson: join(runDir, "run-report.json"),
|
|
38
|
+
artifactManifestJson: join(runDir, "artifact-manifest.json"),
|
|
39
|
+
progressJsonl: join(runDir, "progress.jsonl"),
|
|
40
|
+
trainingDir,
|
|
41
|
+
trainingInputDir: join(trainingDir, "input", "data", "training"),
|
|
42
|
+
trainingConfigDir: join(trainingDir, "input", "config"),
|
|
43
|
+
trainingOutputDir: join(trainingDir, "output"),
|
|
44
|
+
trainingModelDir: join(trainingDir, "model"),
|
|
45
|
+
trainingLog: join(trainingDir, "training.log")
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
async function ensureSafeRunDirectory(artifacts) {
|
|
49
|
+
await mkdir(artifacts.root, { recursive: true });
|
|
50
|
+
const canonicalRoot = await realpath(artifacts.root);
|
|
51
|
+
const pathFromRoot = relative(artifacts.root, artifacts.runDir);
|
|
52
|
+
const segments = pathFromRoot ? pathFromRoot.split(sep).filter(Boolean) : [];
|
|
53
|
+
let current = artifacts.root;
|
|
54
|
+
for (const segment of segments) {
|
|
55
|
+
current = join(current, segment);
|
|
56
|
+
let metadata = await lstat(current).catch((error) => {
|
|
57
|
+
if (error.code === "ENOENT") return null;
|
|
58
|
+
throw error;
|
|
59
|
+
});
|
|
60
|
+
if (!metadata) {
|
|
61
|
+
await mkdir(current).catch((error) => {
|
|
62
|
+
if (error.code !== "EEXIST") throw error;
|
|
63
|
+
});
|
|
64
|
+
metadata = await lstat(current);
|
|
65
|
+
}
|
|
66
|
+
if (metadata.isSymbolicLink()) {
|
|
67
|
+
throw new Error(`Artifact path must not contain symbolic links: ${current}`);
|
|
68
|
+
}
|
|
69
|
+
if (!metadata.isDirectory()) {
|
|
70
|
+
throw new Error(`Artifact path component is not a directory: ${current}`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const canonicalRunDir = await realpath(artifacts.runDir);
|
|
74
|
+
const containment = relative(canonicalRoot, canonicalRunDir);
|
|
75
|
+
if (containment === ".." || containment.startsWith(`..${sep}`) || isAbsolute(containment)) {
|
|
76
|
+
throw new Error(`Artifact run directory escapes artifactRoot: ${artifacts.runDir}`);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
async function claimRunArtifactDirectory(args) {
|
|
80
|
+
await ensureSafeRunDirectory(args.artifacts);
|
|
81
|
+
const ownerPath = join(args.artifacts.runDir, RUN_OWNER_FILE);
|
|
82
|
+
const proposed = {
|
|
83
|
+
schema_version: 1,
|
|
84
|
+
run_id: args.runId,
|
|
85
|
+
user_id: args.userId,
|
|
86
|
+
behavior_spec_id: args.behaviorSpecId,
|
|
87
|
+
created_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
88
|
+
};
|
|
89
|
+
try {
|
|
90
|
+
const handle = await open(ownerPath, "wx", 384);
|
|
91
|
+
try {
|
|
92
|
+
await handle.writeFile(`${JSON.stringify(proposed, null, 2)}
|
|
93
|
+
`);
|
|
94
|
+
await handle.sync();
|
|
95
|
+
} finally {
|
|
96
|
+
await handle.close();
|
|
97
|
+
}
|
|
98
|
+
return proposed;
|
|
99
|
+
} catch (error) {
|
|
100
|
+
if (error.code !== "EEXIST") throw error;
|
|
101
|
+
}
|
|
102
|
+
let existing;
|
|
103
|
+
try {
|
|
104
|
+
existing = JSON.parse(await readFile(ownerPath, "utf8"));
|
|
105
|
+
} catch (error) {
|
|
106
|
+
throw new Error(`Artifact run owner claim is unreadable: ${ownerPath}`, { cause: error });
|
|
107
|
+
}
|
|
108
|
+
if (existing.schema_version !== 1 || existing.run_id !== args.runId || existing.user_id !== args.userId || existing.behavior_spec_id !== args.behaviorSpecId) {
|
|
109
|
+
throw new Error(
|
|
110
|
+
`Artifact directory ${args.artifacts.runDir} is already owned by run ${existing.run_id ?? "unknown"}; run ${args.runId} must use a different artifacts.prefix.`
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
return existing;
|
|
114
|
+
}
|
|
115
|
+
async function prepareRunDirectories(artifacts) {
|
|
116
|
+
await ensureSafeRunDirectory(artifacts);
|
|
117
|
+
await Promise.all([
|
|
118
|
+
mkdir(artifacts.runDir, { recursive: true }),
|
|
119
|
+
mkdir(artifacts.trainingInputDir, { recursive: true }),
|
|
120
|
+
mkdir(artifacts.trainingConfigDir, { recursive: true }),
|
|
121
|
+
mkdir(artifacts.trainingOutputDir, { recursive: true }),
|
|
122
|
+
mkdir(artifacts.trainingModelDir, { recursive: true })
|
|
123
|
+
]);
|
|
124
|
+
}
|
|
125
|
+
async function writeJson(path, value) {
|
|
126
|
+
await mkdir(dirname(path), { recursive: true });
|
|
127
|
+
await writeFile(path, `${JSON.stringify(value, null, 2)}
|
|
128
|
+
`, "utf8");
|
|
129
|
+
}
|
|
130
|
+
async function writeJsonAtomic(path, value) {
|
|
131
|
+
await writeFileAtomic(path, `${JSON.stringify(value, null, 2)}
|
|
132
|
+
`);
|
|
133
|
+
}
|
|
134
|
+
async function writeFileAtomic(path, value) {
|
|
135
|
+
await mkdir(dirname(path), { recursive: true });
|
|
136
|
+
const temporary = `${path}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
|
137
|
+
await writeFile(temporary, value);
|
|
138
|
+
await rename(temporary, path);
|
|
139
|
+
}
|
|
140
|
+
async function readJson(path) {
|
|
141
|
+
return JSON.parse(await readFile(path, "utf8"));
|
|
142
|
+
}
|
|
143
|
+
async function copyDatasetToTrainingChannel(artifacts) {
|
|
144
|
+
const destination = join(artifacts.trainingInputDir, "training.jsonl");
|
|
145
|
+
await copyFile(artifacts.trainingJsonl, destination);
|
|
146
|
+
return destination;
|
|
147
|
+
}
|
|
148
|
+
var MANIFEST_IGNORED_PATHS = /* @__PURE__ */ new Set([
|
|
149
|
+
"artifact-manifest.json",
|
|
150
|
+
// These are append-only/control-plane files rather than immutable stage
|
|
151
|
+
// outputs. Including them would invalidate a manifest whenever the store
|
|
152
|
+
// records the next stage event or refreshes the canonical request copy.
|
|
153
|
+
"progress.jsonl",
|
|
154
|
+
"request.json",
|
|
155
|
+
RUN_OWNER_FILE,
|
|
156
|
+
ARTIFACT_WORKFLOW_LOCK_FILE
|
|
157
|
+
]);
|
|
158
|
+
function portableRelative(root, path) {
|
|
159
|
+
return relative(root, path).split(sep).join("/");
|
|
160
|
+
}
|
|
161
|
+
function shouldManifest(path) {
|
|
162
|
+
const atomicTemporary = /\.\d+\.\d+\.[0-9a-f-]{36}\.tmp$/i.test(path);
|
|
163
|
+
return !MANIFEST_IGNORED_PATHS.has(path) && !atomicTemporary;
|
|
164
|
+
}
|
|
165
|
+
async function listArtifactFiles(root, current = root) {
|
|
166
|
+
const files = [];
|
|
167
|
+
for (const entry of await readdir(current, { withFileTypes: true }).catch(() => [])) {
|
|
168
|
+
const absolute = join(current, entry.name);
|
|
169
|
+
if (entry.isSymbolicLink()) {
|
|
170
|
+
throw new Error(`Artifact tree must not contain symbolic links: ${absolute}`);
|
|
171
|
+
}
|
|
172
|
+
if (entry.isDirectory()) {
|
|
173
|
+
files.push(...await listArtifactFiles(root, absolute));
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (!entry.isFile()) continue;
|
|
177
|
+
const relativePath = portableRelative(root, absolute);
|
|
178
|
+
if (!shouldManifest(relativePath)) continue;
|
|
179
|
+
const metadata = await stat(absolute).catch(() => null);
|
|
180
|
+
if (metadata?.isFile()) files.push(relativePath);
|
|
181
|
+
}
|
|
182
|
+
return files.sort();
|
|
183
|
+
}
|
|
184
|
+
async function sha256File(path) {
|
|
185
|
+
const hash = createHash("sha256");
|
|
186
|
+
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
|
187
|
+
return hash.digest("hex");
|
|
188
|
+
}
|
|
189
|
+
async function describeArtifactFile(root, path) {
|
|
190
|
+
const absolute = join(root, path);
|
|
191
|
+
const metadata = await stat(absolute);
|
|
192
|
+
return {
|
|
193
|
+
path,
|
|
194
|
+
size_bytes: metadata.size,
|
|
195
|
+
sha256: await sha256File(absolute)
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
function assertManifestFiles(value, label) {
|
|
199
|
+
if (!Array.isArray(value)) throw new Error(`Artifact manifest ${label} must be an array.`);
|
|
200
|
+
const paths = /* @__PURE__ */ new Set();
|
|
201
|
+
for (const file of value) {
|
|
202
|
+
if (!file || typeof file.path !== "string" || !Number.isSafeInteger(file.size_bytes) || file.size_bytes < 0 || typeof file.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(file.sha256)) {
|
|
203
|
+
throw new Error(`Artifact manifest contains an invalid ${label} entry.`);
|
|
204
|
+
}
|
|
205
|
+
const manifestPath = file.path;
|
|
206
|
+
const segments = manifestPath.split("/");
|
|
207
|
+
if (!manifestPath || manifestPath.includes("\\") || manifestPath.includes("\0") || manifestPath.startsWith("/") || /^[a-z]:/i.test(manifestPath) || segments.some((segment) => !segment || segment === "." || segment === "..")) {
|
|
208
|
+
throw new Error(`Artifact manifest contains an unsafe ${label} path: ${manifestPath}`);
|
|
209
|
+
}
|
|
210
|
+
if (paths.has(manifestPath)) throw new Error(`Artifact manifest contains a duplicate ${label} path: ${manifestPath}`);
|
|
211
|
+
paths.add(manifestPath);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
function hasPeftAdapterFiles(files) {
|
|
215
|
+
const names = new Set(
|
|
216
|
+
files.filter((file) => file.size_bytes > 0).map((file) => file.path.split("/").at(-1)?.toLowerCase())
|
|
217
|
+
);
|
|
218
|
+
return names.has("adapter_config.json") && (names.has("adapter_model.safetensors") || names.has("adapter_model.bin"));
|
|
219
|
+
}
|
|
220
|
+
function parseArtifactManifest(value) {
|
|
221
|
+
if (!value || typeof value !== "object") throw new Error("Artifact manifest must be an object.");
|
|
222
|
+
const candidate = value;
|
|
223
|
+
if (candidate.schema_version !== 1 || !Array.isArray(candidate.files)) {
|
|
224
|
+
throw new Error("Unsupported or invalid artifact manifest.");
|
|
225
|
+
}
|
|
226
|
+
assertManifestFiles(candidate.files, "file");
|
|
227
|
+
if (candidate.model) {
|
|
228
|
+
const model = candidate.model;
|
|
229
|
+
const allowedModelKeys = /* @__PURE__ */ new Set([
|
|
230
|
+
"artifact_kind",
|
|
231
|
+
"format",
|
|
232
|
+
"framework",
|
|
233
|
+
"base_model",
|
|
234
|
+
"base_model_revision",
|
|
235
|
+
"base_model_artifact_uri",
|
|
236
|
+
"base_model_fingerprint",
|
|
237
|
+
"artifact_uri",
|
|
238
|
+
"artifact_root",
|
|
239
|
+
"servable",
|
|
240
|
+
"files"
|
|
241
|
+
]);
|
|
242
|
+
const hasBaseArtifact = model.base_model_artifact_uri !== void 0;
|
|
243
|
+
const hasBaseFingerprint = model.base_model_fingerprint !== void 0;
|
|
244
|
+
if (Object.keys(model).some((key) => !allowedModelKeys.has(key)) || model.artifact_kind !== "file" && model.artifact_kind !== "directory" || model.format !== "tar.gz" && model.format !== "huggingface-directory" || model.framework !== "transformers-peft" || typeof model.base_model !== "string" || model.base_model_revision !== void 0 && typeof model.base_model_revision !== "string" || model.base_model_artifact_uri !== void 0 && typeof model.base_model_artifact_uri !== "string" || model.base_model_fingerprint !== void 0 && typeof model.base_model_fingerprint !== "string" || hasBaseArtifact !== hasBaseFingerprint || typeof model.artifact_uri !== "string" || typeof model.artifact_root !== "string" || model.servable !== true || !Array.isArray(model.files) || model.files.length === 0 || model.artifact_kind === "file" && model.format !== "tar.gz" || model.artifact_kind === "directory" && model.format !== "huggingface-directory") {
|
|
245
|
+
throw new Error("Artifact manifest contains invalid model contract metadata.");
|
|
246
|
+
}
|
|
247
|
+
assertManifestFiles(model.files, "model file");
|
|
248
|
+
if (model.artifact_kind === "directory" && !hasPeftAdapterFiles(model.files)) {
|
|
249
|
+
throw new Error("PEFT model manifest requires adapter weights and adapter_config.json.");
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return candidate;
|
|
253
|
+
}
|
|
254
|
+
async function writeArtifactManifest(artifacts, options = {}) {
|
|
255
|
+
const paths = await listArtifactFiles(artifacts.runDir);
|
|
256
|
+
let model;
|
|
257
|
+
if (options.model) {
|
|
258
|
+
if (options.model.framework !== "transformers-peft" || options.model.servable !== true || options.model.artifact_kind === "file" && options.model.format !== "tar.gz" || options.model.artifact_kind === "directory" && options.model.format !== "huggingface-directory") {
|
|
259
|
+
throw new Error("TT Local model artifacts must be verified, servable Transformers PEFT adapters.");
|
|
260
|
+
}
|
|
261
|
+
if (options.model.base_model_artifact_uri === void 0 !== (options.model.base_model_fingerprint === void 0)) {
|
|
262
|
+
throw new Error("Model contracts must record a local base-model URI and fingerprint together.");
|
|
263
|
+
}
|
|
264
|
+
const modelRoot = resolve(options.model.artifact_root);
|
|
265
|
+
const modelPaths = options.model.artifact_kind === "file" ? [portableRelative(dirname(modelRoot), modelRoot)] : await listArtifactFiles(modelRoot);
|
|
266
|
+
const checksumRoot = options.model.artifact_kind === "file" ? dirname(modelRoot) : modelRoot;
|
|
267
|
+
if (options.model.artifact_kind === "file") {
|
|
268
|
+
if (!modelRoot.toLowerCase().endsWith(".tar.gz")) {
|
|
269
|
+
throw new Error(`PEFT file artifacts must be .tar.gz archives: ${modelRoot}`);
|
|
270
|
+
}
|
|
271
|
+
const archive = await verifyTarGzipArchive(modelRoot);
|
|
272
|
+
if (archive.adapter_weight_entries === 0 || archive.adapter_weight_bytes === 0) {
|
|
273
|
+
throw new Error(`PEFT model archive contains no adapter_model.safetensors or adapter_model.bin: ${modelRoot}`);
|
|
274
|
+
}
|
|
275
|
+
if (archive.adapter_config_entries === 0) {
|
|
276
|
+
throw new Error(`PEFT model archive contains no adapter_config.json: ${modelRoot}`);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
const modelFiles = await Promise.all(modelPaths.map((path) => describeArtifactFile(checksumRoot, path)));
|
|
280
|
+
if (options.model.artifact_kind === "directory" && !hasPeftAdapterFiles(modelFiles)) {
|
|
281
|
+
throw new Error(`PEFT model directory requires adapter weights and adapter_config.json: ${modelRoot}`);
|
|
282
|
+
}
|
|
283
|
+
model = {
|
|
284
|
+
...options.model,
|
|
285
|
+
artifact_root: modelRoot,
|
|
286
|
+
files: modelFiles
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
const manifest = {
|
|
290
|
+
schema_version: 1,
|
|
291
|
+
generated_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
292
|
+
files: await Promise.all(paths.map((path) => describeArtifactFile(artifacts.runDir, path))),
|
|
293
|
+
...model ? { model } : {}
|
|
294
|
+
};
|
|
295
|
+
await writeJsonAtomic(artifacts.artifactManifestJson, manifest);
|
|
296
|
+
return manifest;
|
|
297
|
+
}
|
|
298
|
+
async function verifyArtifactManifest(manifestPath, options = {}) {
|
|
299
|
+
const root = dirname(resolve(manifestPath));
|
|
300
|
+
const manifest = parseArtifactManifest(JSON.parse(await readFile(manifestPath, "utf8")));
|
|
301
|
+
const expected = new Map(manifest.files.map((file) => [file.path, file]));
|
|
302
|
+
const actualPaths = await listArtifactFiles(root);
|
|
303
|
+
const actual = new Set(actualPaths);
|
|
304
|
+
const missing = [];
|
|
305
|
+
const changed = [];
|
|
306
|
+
const normalizedRequired = (options.requiredPaths ?? []).map((path) => path.split(sep).join("/").replace(/^\.\//, ""));
|
|
307
|
+
const checkedEntries = options.scopeToRequired ? normalizedRequired.flatMap((path) => {
|
|
308
|
+
const entry = expected.get(path);
|
|
309
|
+
return entry ? [[path, entry]] : [];
|
|
310
|
+
}) : [...expected.entries()];
|
|
311
|
+
for (const [path, expectedFile] of checkedEntries) {
|
|
312
|
+
if (!actual.has(path)) {
|
|
313
|
+
missing.push(path);
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
const actualFile = await describeArtifactFile(root, path);
|
|
317
|
+
if (actualFile.size_bytes !== expectedFile.size_bytes || actualFile.sha256 !== expectedFile.sha256) {
|
|
318
|
+
changed.push(path);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
for (const normalized of normalizedRequired) {
|
|
322
|
+
if (!expected.has(normalized) && !missing.includes(normalized)) missing.push(normalized);
|
|
323
|
+
}
|
|
324
|
+
const unexpected = options.allowUnexpected || options.scopeToRequired ? [] : actualPaths.filter((path) => !expected.has(path));
|
|
325
|
+
let modelChecked = 0;
|
|
326
|
+
if (manifest.model && (options.verifyModel ?? !options.scopeToRequired)) {
|
|
327
|
+
const modelRoot = resolve(manifest.model.artifact_root);
|
|
328
|
+
const checksumRoot = manifest.model.artifact_kind === "file" ? dirname(modelRoot) : modelRoot;
|
|
329
|
+
const modelActualPaths = manifest.model.artifact_kind === "file" ? [portableRelative(checksumRoot, modelRoot)] : await listArtifactFiles(modelRoot);
|
|
330
|
+
const modelActual = new Set(modelActualPaths);
|
|
331
|
+
for (const expectedFile of manifest.model.files) {
|
|
332
|
+
const label = `model:${expectedFile.path}`;
|
|
333
|
+
if (!modelActual.has(expectedFile.path)) {
|
|
334
|
+
missing.push(label);
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
const actualFile = await describeArtifactFile(checksumRoot, expectedFile.path);
|
|
338
|
+
modelChecked += 1;
|
|
339
|
+
if (actualFile.size_bytes !== expectedFile.size_bytes || actualFile.sha256 !== expectedFile.sha256) {
|
|
340
|
+
changed.push(label);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
if (!options.allowUnexpected) {
|
|
344
|
+
const expectedModelPaths = new Set(manifest.model.files.map((file) => file.path));
|
|
345
|
+
unexpected.push(...modelActualPaths.filter((path) => !expectedModelPaths.has(path)).map((path) => `model:${path}`));
|
|
346
|
+
}
|
|
347
|
+
if (manifest.model.artifact_kind === "file") {
|
|
348
|
+
try {
|
|
349
|
+
const archive = await verifyTarGzipArchive(modelRoot);
|
|
350
|
+
if (archive.adapter_weight_entries === 0 || archive.adapter_weight_bytes === 0) {
|
|
351
|
+
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_adapter_weights`);
|
|
352
|
+
}
|
|
353
|
+
if (archive.adapter_config_entries === 0) {
|
|
354
|
+
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:missing_adapter_config`);
|
|
355
|
+
}
|
|
356
|
+
} catch {
|
|
357
|
+
changed.push(`model:${portableRelative(checksumRoot, modelRoot)}:invalid_archive`);
|
|
358
|
+
}
|
|
359
|
+
} else if (!hasPeftAdapterFiles(
|
|
360
|
+
await Promise.all(modelActualPaths.map((path) => describeArtifactFile(checksumRoot, path)))
|
|
361
|
+
)) {
|
|
362
|
+
changed.push("model:missing_peft_adapter");
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
return {
|
|
366
|
+
valid: missing.length === 0 && changed.length === 0 && unexpected.length === 0,
|
|
367
|
+
checked: checkedEntries.length + modelChecked,
|
|
368
|
+
missing: [...new Set(missing)].sort(),
|
|
369
|
+
changed: changed.sort(),
|
|
370
|
+
unexpected: unexpected.sort()
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
async function verifyTarGzipArchive(path) {
|
|
374
|
+
const maxEntries = 1e5;
|
|
375
|
+
const maxExpandedBytes = 20 * 1024 * 1024 * 1024;
|
|
376
|
+
const maxPaxBytes = 1024 * 1024;
|
|
377
|
+
const safeMemberPath = (value) => {
|
|
378
|
+
const normalized = value.replaceAll("\\", "/");
|
|
379
|
+
const segments = normalized.split("/").filter(Boolean);
|
|
380
|
+
if (!normalized || normalized.startsWith("/") || /^[a-z]:\//i.test(normalized) || segments.includes("..")) {
|
|
381
|
+
throw new Error(`Unsafe tar member path ${JSON.stringify(normalized)} in ${path}.`);
|
|
382
|
+
}
|
|
383
|
+
return normalized;
|
|
384
|
+
};
|
|
385
|
+
const parsePax = (payload, global) => {
|
|
386
|
+
let offset = 0;
|
|
387
|
+
let memberPath;
|
|
388
|
+
const allowedMetadata = /* @__PURE__ */ new Set(["mtime", "atime", "ctime", "uid", "gid", "uname", "gname"]);
|
|
389
|
+
while (offset < payload.length) {
|
|
390
|
+
const separator = payload.indexOf(32, offset);
|
|
391
|
+
if (separator < 0) throw new Error(`Malformed PAX record in ${path}.`);
|
|
392
|
+
const lengthText = payload.subarray(offset, separator).toString("ascii");
|
|
393
|
+
if (!/^[1-9][0-9]*$/.test(lengthText)) throw new Error(`Malformed PAX record length in ${path}.`);
|
|
394
|
+
const length = Number.parseInt(lengthText, 10);
|
|
395
|
+
const end = offset + length;
|
|
396
|
+
if (!Number.isSafeInteger(length) || end > payload.length || payload[end - 1] !== 10) {
|
|
397
|
+
throw new Error(`Truncated PAX record in ${path}.`);
|
|
398
|
+
}
|
|
399
|
+
const record = payload.subarray(separator + 1, end - 1).toString("utf8");
|
|
400
|
+
const equals = record.indexOf("=");
|
|
401
|
+
if (equals <= 0) throw new Error(`Malformed PAX key/value in ${path}.`);
|
|
402
|
+
const key = record.slice(0, equals);
|
|
403
|
+
const value = record.slice(equals + 1);
|
|
404
|
+
if (key === "path") {
|
|
405
|
+
if (global) throw new Error(`Global PAX path overrides are unsupported in ${path}.`);
|
|
406
|
+
memberPath = safeMemberPath(value);
|
|
407
|
+
} else if (!allowedMetadata.has(key)) {
|
|
408
|
+
throw new Error(`Unsupported PAX key ${JSON.stringify(key)} in ${path}.`);
|
|
409
|
+
}
|
|
410
|
+
offset = end;
|
|
411
|
+
}
|
|
412
|
+
return memberPath;
|
|
413
|
+
};
|
|
414
|
+
const gunzip = createReadStream(path).pipe(createGunzip());
|
|
415
|
+
let buffered = Buffer.alloc(0);
|
|
416
|
+
let payloadBytes = 0;
|
|
417
|
+
let paddingBytes = 0;
|
|
418
|
+
let paxType = null;
|
|
419
|
+
let paxChunks = [];
|
|
420
|
+
let pendingPaxPath;
|
|
421
|
+
let entries = 0;
|
|
422
|
+
let payloadEntries = 0;
|
|
423
|
+
let recognizedPayloadEntries = 0;
|
|
424
|
+
let recognizedPayloadBytes = 0;
|
|
425
|
+
let adapterWeightEntries = 0;
|
|
426
|
+
let adapterWeightBytes = 0;
|
|
427
|
+
let fullModelWeightEntries = 0;
|
|
428
|
+
let fullModelWeightBytes = 0;
|
|
429
|
+
let adapterConfigEntries = 0;
|
|
430
|
+
let expandedBytes = 0;
|
|
431
|
+
let zeroBlocks = 0;
|
|
432
|
+
let ended = false;
|
|
433
|
+
for await (const chunk of gunzip) {
|
|
434
|
+
buffered = Buffer.concat([buffered, Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)]);
|
|
435
|
+
while (true) {
|
|
436
|
+
if (payloadBytes > 0) {
|
|
437
|
+
if (buffered.length === 0) break;
|
|
438
|
+
const consumed = Math.min(payloadBytes, buffered.length);
|
|
439
|
+
if (paxType) paxChunks.push(buffered.subarray(0, consumed));
|
|
440
|
+
buffered = buffered.subarray(consumed);
|
|
441
|
+
payloadBytes -= consumed;
|
|
442
|
+
continue;
|
|
443
|
+
}
|
|
444
|
+
if (paddingBytes > 0) {
|
|
445
|
+
if (buffered.length === 0) break;
|
|
446
|
+
const consumed = Math.min(paddingBytes, buffered.length);
|
|
447
|
+
buffered = buffered.subarray(consumed);
|
|
448
|
+
paddingBytes -= consumed;
|
|
449
|
+
continue;
|
|
450
|
+
}
|
|
451
|
+
if (paxType) {
|
|
452
|
+
const paxPath = parsePax(Buffer.concat(paxChunks), paxType === "g");
|
|
453
|
+
if (paxType === "x") pendingPaxPath = paxPath;
|
|
454
|
+
paxType = null;
|
|
455
|
+
paxChunks = [];
|
|
456
|
+
continue;
|
|
457
|
+
}
|
|
458
|
+
if (buffered.length < 512) break;
|
|
459
|
+
const header = buffered.subarray(0, 512);
|
|
460
|
+
buffered = buffered.subarray(512);
|
|
461
|
+
if (header.every((byte) => byte === 0)) {
|
|
462
|
+
zeroBlocks += 1;
|
|
463
|
+
if (zeroBlocks >= 2) ended = true;
|
|
464
|
+
continue;
|
|
465
|
+
}
|
|
466
|
+
if (ended) throw new Error(`Tar archive contains data after its end marker: ${path}.`);
|
|
467
|
+
zeroBlocks = 0;
|
|
468
|
+
const checksumText = header.subarray(148, 156).toString("ascii").replace(/\0/g, "").trim();
|
|
469
|
+
if (!/^[0-7]+$/.test(checksumText)) throw new Error(`Invalid tar checksum field in ${path}.`);
|
|
470
|
+
const expectedChecksum = Number.parseInt(checksumText, 8);
|
|
471
|
+
let actualChecksum = 0;
|
|
472
|
+
for (let index = 0; index < header.length; index += 1) {
|
|
473
|
+
actualChecksum += index >= 148 && index < 156 ? 32 : header[index];
|
|
474
|
+
}
|
|
475
|
+
if (actualChecksum !== expectedChecksum) throw new Error(`Tar member checksum mismatch in ${path}.`);
|
|
476
|
+
const sizeText = header.subarray(124, 136).toString("ascii").replace(/\0/g, "").trim();
|
|
477
|
+
if (sizeText && !/^[0-7]+$/.test(sizeText)) throw new Error(`Invalid tar member size in ${path}.`);
|
|
478
|
+
const size = sizeText ? Number.parseInt(sizeText, 8) : 0;
|
|
479
|
+
if (!Number.isSafeInteger(size) || size < 0) throw new Error(`Invalid tar member size in ${path}.`);
|
|
480
|
+
const type = String.fromCharCode(header[156] ?? 0);
|
|
481
|
+
if (type !== "\0" && type !== "0" && type !== "5" && type !== "x" && type !== "g") {
|
|
482
|
+
throw new Error(`Unsafe or unsupported tar member type ${JSON.stringify(type)} in ${path}.`);
|
|
483
|
+
}
|
|
484
|
+
const shortName = header.subarray(0, 100).toString("utf8").replace(/\0.*$/, "");
|
|
485
|
+
const prefix = header.subarray(345, 500).toString("utf8").replace(/\0.*$/, "");
|
|
486
|
+
const headerPath = safeMemberPath(prefix ? `${prefix}/${shortName}` : shortName);
|
|
487
|
+
const memberPath = type === "x" || type === "g" ? headerPath : safeMemberPath(pendingPaxPath ?? headerPath);
|
|
488
|
+
if (type !== "x" && type !== "g") pendingPaxPath = void 0;
|
|
489
|
+
const segments = memberPath.split("/").filter(Boolean);
|
|
490
|
+
const name = segments.at(-1)?.toLowerCase();
|
|
491
|
+
const metadataOnly = name === "training-metrics.json" || name === "trainer_state.json" || name === "training_args.bin" || name === "config.json" || name === "generation_config.json" || name === "adapter_config.json" || name === "readme.md" || name?.startsWith("tokenizer") || name?.startsWith("special_tokens") || name?.startsWith("added_tokens") || name === "vocab.json" || name === "merges.txt";
|
|
492
|
+
const recognizedPayload = Boolean(name) && name !== "training_args.bin" && (name.endsWith(".safetensors") || name.endsWith(".bin"));
|
|
493
|
+
const adapterWeight = name === "adapter_model.safetensors" || name === "adapter_model.bin";
|
|
494
|
+
const fullModelWeight = Boolean(name) && (name.endsWith(".safetensors") && !name.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(name));
|
|
495
|
+
if (size > 0 && name === "adapter_config.json" && (type === "\0" || type === "0")) {
|
|
496
|
+
adapterConfigEntries += 1;
|
|
497
|
+
}
|
|
498
|
+
if ((type === "x" || type === "g") && size > maxPaxBytes) {
|
|
499
|
+
throw new Error(`PAX metadata exceeds 1 MiB in ${path}.`);
|
|
500
|
+
}
|
|
501
|
+
if (size > 0 && !metadataOnly && (type === "\0" || type === "0")) payloadEntries += 1;
|
|
502
|
+
if (size > 0 && recognizedPayload && (type === "\0" || type === "0")) {
|
|
503
|
+
recognizedPayloadEntries += 1;
|
|
504
|
+
recognizedPayloadBytes += size;
|
|
505
|
+
}
|
|
506
|
+
if (size > 0 && adapterWeight && (type === "\0" || type === "0")) {
|
|
507
|
+
adapterWeightEntries += 1;
|
|
508
|
+
adapterWeightBytes += size;
|
|
509
|
+
}
|
|
510
|
+
if (size > 0 && fullModelWeight && (type === "\0" || type === "0")) {
|
|
511
|
+
fullModelWeightEntries += 1;
|
|
512
|
+
fullModelWeightBytes += size;
|
|
513
|
+
}
|
|
514
|
+
expandedBytes += size;
|
|
515
|
+
if (expandedBytes > maxExpandedBytes) throw new Error(`Tar archive exceeds 20 GiB expanded size: ${path}.`);
|
|
516
|
+
payloadBytes = size;
|
|
517
|
+
paddingBytes = (512 - size % 512) % 512;
|
|
518
|
+
paxType = type === "x" || type === "g" ? type : null;
|
|
519
|
+
paxChunks = [];
|
|
520
|
+
entries += 1;
|
|
521
|
+
if (entries > maxEntries) throw new Error(`Tar archive exceeds ${maxEntries} members: ${path}.`);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
if (payloadBytes !== 0 || paddingBytes !== 0 || paxType || buffered.some((byte) => byte !== 0)) {
|
|
525
|
+
throw new Error(`Truncated tar archive: ${path}.`);
|
|
526
|
+
}
|
|
527
|
+
if (pendingPaxPath) throw new Error(`PAX path override has no following member in ${path}.`);
|
|
528
|
+
if (!ended) throw new Error(`Tar archive is missing its end marker: ${path}.`);
|
|
529
|
+
if (entries === 0) throw new Error(`Empty tar archive: ${path}.`);
|
|
530
|
+
if (payloadEntries === 0) throw new Error(`Tar archive contains no model payload files: ${path}.`);
|
|
531
|
+
return {
|
|
532
|
+
entries,
|
|
533
|
+
payload_entries: payloadEntries,
|
|
534
|
+
recognized_payload_entries: recognizedPayloadEntries,
|
|
535
|
+
recognized_payload_bytes: recognizedPayloadBytes,
|
|
536
|
+
adapter_weight_entries: adapterWeightEntries,
|
|
537
|
+
adapter_weight_bytes: adapterWeightBytes,
|
|
538
|
+
full_model_weight_entries: fullModelWeightEntries,
|
|
539
|
+
full_model_weight_bytes: fullModelWeightBytes,
|
|
540
|
+
adapter_config_entries: adapterConfigEntries,
|
|
541
|
+
expanded_bytes: expandedBytes
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
async function assertArtifactManifest(manifestPath, options = {}) {
|
|
545
|
+
const result = await verifyArtifactManifest(manifestPath, options);
|
|
546
|
+
if (!result.valid) {
|
|
547
|
+
const details = [
|
|
548
|
+
result.missing.length ? `missing=${result.missing.join(",")}` : "",
|
|
549
|
+
result.changed.length ? `changed=${result.changed.join(",")}` : "",
|
|
550
|
+
result.unexpected.length ? `unexpected=${result.unexpected.join(",")}` : ""
|
|
551
|
+
].filter(Boolean).join(" ");
|
|
552
|
+
throw new Error(`Artifact integrity verification failed${details ? `: ${details}` : "."}`);
|
|
553
|
+
}
|
|
554
|
+
return result;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
// src/local-runtime/model-registry.ts
|
|
558
|
+
import { lstat as lstat2, readdir as readdir2 } from "fs/promises";
|
|
559
|
+
import { resolve as resolve2 } from "path";
|
|
560
|
+
import { fileURLToPath } from "url";
|
|
561
|
+
var TRAINING_MODELS = [
|
|
562
|
+
{
|
|
563
|
+
id: "Qwen/Qwen3.5-2B",
|
|
564
|
+
family: "qwen3_5",
|
|
565
|
+
defaultLearningRate: 1e-5,
|
|
566
|
+
defaultPerDeviceBatchSize: 1,
|
|
567
|
+
defaultGradientAccumulationSteps: 8,
|
|
568
|
+
defaultLoraRank: 16,
|
|
569
|
+
defaultLoraAlpha: 32,
|
|
570
|
+
defaultLoraDropout: 0.05,
|
|
571
|
+
defaultMaxSeqLength: 2048
|
|
572
|
+
}
|
|
573
|
+
];
|
|
574
|
+
function resolveTrainingModel(modelId) {
|
|
575
|
+
const normalized = modelId.trim().toLowerCase();
|
|
576
|
+
const model = TRAINING_MODELS.find(
|
|
577
|
+
(candidate) => candidate.id.toLowerCase() === normalized
|
|
578
|
+
);
|
|
579
|
+
if (!model) {
|
|
580
|
+
throw new Error(
|
|
581
|
+
`Unsupported base model "${modelId}". Supported model: ${TRAINING_MODELS[0].id}`
|
|
582
|
+
);
|
|
583
|
+
}
|
|
584
|
+
return model;
|
|
585
|
+
}
|
|
586
|
+
function canonicalizeTrainingModel(modelId) {
|
|
587
|
+
return resolveTrainingModel(modelId).id;
|
|
588
|
+
}
|
|
589
|
+
var CERTIFIED_TEXT_CONFIG = {
|
|
590
|
+
model_type: "qwen3_5_text",
|
|
591
|
+
hidden_size: 2048,
|
|
592
|
+
num_hidden_layers: 24,
|
|
593
|
+
num_attention_heads: 8,
|
|
594
|
+
num_key_value_heads: 2,
|
|
595
|
+
intermediate_size: 6144,
|
|
596
|
+
vocab_size: 248320
|
|
597
|
+
};
|
|
598
|
+
function assertCertifiedBaseModelConfig(value, label = "base-model config.json") {
|
|
599
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
600
|
+
throw new Error(`${label} must contain a JSON object.`);
|
|
601
|
+
}
|
|
602
|
+
const config = value;
|
|
603
|
+
const architectures = config.architectures;
|
|
604
|
+
const textConfig = config.text_config;
|
|
605
|
+
if (config.model_type !== "qwen3_5" || !Array.isArray(architectures) || !architectures.includes("Qwen3_5ForConditionalGeneration") || !textConfig || typeof textConfig !== "object" || Array.isArray(textConfig)) {
|
|
606
|
+
throw new Error(
|
|
607
|
+
`${label} is not the certified Qwen/Qwen3.5-2B architecture.`
|
|
608
|
+
);
|
|
609
|
+
}
|
|
610
|
+
const text = textConfig;
|
|
611
|
+
for (const [key, expected] of Object.entries(CERTIFIED_TEXT_CONFIG)) {
|
|
612
|
+
if (text[key] !== expected) {
|
|
613
|
+
throw new Error(
|
|
614
|
+
`${label} is not the certified Qwen/Qwen3.5-2B architecture: text_config.${key} must be ${JSON.stringify(expected)}.`
|
|
615
|
+
);
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
function localModelArtifactPath(uri) {
|
|
620
|
+
if (uri.startsWith("file://")) return fileURLToPath(new URL(uri));
|
|
621
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(uri)) {
|
|
622
|
+
throw new Error(`Model artifact must be a local path or file URI: ${uri}`);
|
|
623
|
+
}
|
|
624
|
+
return resolve2(uri);
|
|
625
|
+
}
|
|
626
|
+
function adapterWeightName(name) {
|
|
627
|
+
const lower = name.toLowerCase();
|
|
628
|
+
return lower === "adapter_model.safetensors" || lower === "adapter_model.bin";
|
|
629
|
+
}
|
|
630
|
+
async function inspectAdapterDirectory(path) {
|
|
631
|
+
let count = 0;
|
|
632
|
+
let bytes = 0;
|
|
633
|
+
let hasConfig = false;
|
|
634
|
+
for (const entry of await readdir2(path, { withFileTypes: true })) {
|
|
635
|
+
const child = resolve2(path, entry.name);
|
|
636
|
+
if (entry.isSymbolicLink()) {
|
|
637
|
+
throw new Error(`Model artifact must not contain symbolic links: ${child}`);
|
|
638
|
+
}
|
|
639
|
+
if (entry.isDirectory()) {
|
|
640
|
+
const nested = await inspectAdapterDirectory(child);
|
|
641
|
+
count += nested.count;
|
|
642
|
+
bytes += nested.bytes;
|
|
643
|
+
hasConfig ||= nested.hasConfig;
|
|
644
|
+
continue;
|
|
645
|
+
}
|
|
646
|
+
if (!entry.isFile()) continue;
|
|
647
|
+
const metadata = await lstat2(child);
|
|
648
|
+
if (adapterWeightName(entry.name) && metadata.size > 0) {
|
|
649
|
+
count += 1;
|
|
650
|
+
bytes += metadata.size;
|
|
651
|
+
}
|
|
652
|
+
if (entry.name.toLowerCase() === "adapter_config.json" && metadata.size > 0) {
|
|
653
|
+
hasConfig = true;
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
return { count, bytes, hasConfig };
|
|
657
|
+
}
|
|
658
|
+
async function inspectModelArtifact(uri) {
|
|
659
|
+
const path = localModelArtifactPath(uri);
|
|
660
|
+
const metadata = await lstat2(path).catch(() => null);
|
|
661
|
+
if (!metadata) throw new Error(`Model artifact does not exist: ${path}`);
|
|
662
|
+
if (metadata.isSymbolicLink()) {
|
|
663
|
+
throw new Error(`Model artifact must not be a symbolic link: ${path}`);
|
|
664
|
+
}
|
|
665
|
+
if (metadata.isDirectory()) {
|
|
666
|
+
const adapter = await inspectAdapterDirectory(path);
|
|
667
|
+
return {
|
|
668
|
+
uri,
|
|
669
|
+
path,
|
|
670
|
+
kind: "directory",
|
|
671
|
+
adapter_weight_file_count: adapter.count,
|
|
672
|
+
adapter_weight_bytes: adapter.bytes,
|
|
673
|
+
has_adapter_config: adapter.hasConfig
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
if (!metadata.isFile() || !path.toLowerCase().endsWith(".tar.gz")) {
|
|
677
|
+
throw new Error(
|
|
678
|
+
`Model artifact must be a PEFT adapter directory or .tar.gz archive: ${path}`
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
const archive = await verifyTarGzipArchive(path);
|
|
682
|
+
return {
|
|
683
|
+
uri,
|
|
684
|
+
path,
|
|
685
|
+
kind: "file",
|
|
686
|
+
adapter_weight_file_count: archive.adapter_weight_entries,
|
|
687
|
+
adapter_weight_bytes: archive.adapter_weight_bytes,
|
|
688
|
+
has_adapter_config: archive.adapter_config_entries > 0
|
|
689
|
+
};
|
|
690
|
+
}
|
|
691
|
+
async function assertUsableModelArtifact(uri) {
|
|
692
|
+
const inspection = await inspectModelArtifact(uri);
|
|
693
|
+
if (inspection.adapter_weight_file_count === 0 || inspection.adapter_weight_bytes === 0) {
|
|
694
|
+
throw new Error(
|
|
695
|
+
`PEFT model artifact ${inspection.path} contains no non-empty adapter_model weights.`
|
|
696
|
+
);
|
|
697
|
+
}
|
|
698
|
+
if (!inspection.has_adapter_config) {
|
|
699
|
+
throw new Error(
|
|
700
|
+
`PEFT model artifact ${inspection.path} contains adapter weights but no non-empty adapter_config.json.`
|
|
701
|
+
);
|
|
702
|
+
}
|
|
703
|
+
return inspection;
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
// src/local-runtime/contracts.ts
|
|
707
|
+
import { z } from "zod";
|
|
708
|
+
var datasetFormatSchema = z.literal("chat_jsonl");
|
|
709
|
+
var baseModelRevisionSchema = z.string().regex(/^[0-9a-f]{40}$/i, "base_model_revision must be a 40-character Hugging Face commit SHA").transform((value) => value.toLowerCase());
|
|
710
|
+
var behaviorSpecExampleSchema = z.object({
|
|
711
|
+
input: z.string().min(1),
|
|
712
|
+
output: z.string().min(1)
|
|
713
|
+
}).strict();
|
|
714
|
+
var specSnapshotSchema = z.object({
|
|
715
|
+
name: z.string().min(1),
|
|
716
|
+
description: z.string().default(""),
|
|
717
|
+
system_prompt: z.string().default(""),
|
|
718
|
+
guidelines: z.array(z.string()).default([]),
|
|
719
|
+
examples: z.array(behaviorSpecExampleSchema).default([]),
|
|
720
|
+
constraints: z.array(z.string()).default([]),
|
|
721
|
+
base_model: z.string().transform((value) => canonicalizeTrainingModel(value))
|
|
722
|
+
}).strict();
|
|
723
|
+
var datasetPrebuiltSchema = z.object({
|
|
724
|
+
training: z.string().min(1),
|
|
725
|
+
validation: z.string().min(1).optional(),
|
|
726
|
+
test: z.string().min(1).optional(),
|
|
727
|
+
format: datasetFormatSchema.default("chat_jsonl")
|
|
728
|
+
}).strict().superRefine((dataset, context) => {
|
|
729
|
+
if (!dataset.validation && !dataset.test) {
|
|
730
|
+
context.addIssue({
|
|
731
|
+
code: z.ZodIssueCode.custom,
|
|
732
|
+
path: ["validation"],
|
|
733
|
+
message: "Provide a distinct validation or test split for held-out evaluation"
|
|
734
|
+
});
|
|
735
|
+
}
|
|
736
|
+
});
|
|
737
|
+
var fineTuneHyperparametersSchema = z.object({
|
|
738
|
+
n_epochs: z.number().int().min(1).max(20).default(1),
|
|
739
|
+
learning_rate: z.number().positive().optional(),
|
|
740
|
+
batch_size: z.number().int().min(1).optional(),
|
|
741
|
+
lora_rank: z.number().int().min(1).optional(),
|
|
742
|
+
lora_alpha: z.number().int().min(1).optional(),
|
|
743
|
+
lora_dropout: z.number().min(0).max(1).optional(),
|
|
744
|
+
max_seq_length: z.number().int().min(128).max(32768).optional(),
|
|
745
|
+
gradient_accumulation_steps: z.number().int().min(1).optional(),
|
|
746
|
+
max_eval_examples: z.number().int().min(1).optional(),
|
|
747
|
+
/** Immutable Hugging Face commit used by every stage of the run. */
|
|
748
|
+
base_model_revision: baseModelRevisionSchema.optional()
|
|
749
|
+
}).strict();
|
|
750
|
+
var fineTuneRunRequestSchema = z.object({
|
|
751
|
+
run_id: z.string().uuid(),
|
|
752
|
+
user_id: z.string().min(1),
|
|
753
|
+
behavior_spec_id: z.string().uuid(),
|
|
754
|
+
run_number: z.number().int().min(1),
|
|
755
|
+
spec_snapshot: specSnapshotSchema,
|
|
756
|
+
hyperparameters: fineTuneHyperparametersSchema.default({ n_epochs: 1 }),
|
|
757
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
758
|
+
}).strict().superRefine((request, context) => {
|
|
759
|
+
if (request.spec_snapshot.examples.length === 0 && !request.dataset_prebuilt) {
|
|
760
|
+
context.addIssue({
|
|
761
|
+
code: z.ZodIssueCode.custom,
|
|
762
|
+
path: ["spec_snapshot", "examples"],
|
|
763
|
+
message: "Add examples or provide dataset_prebuilt training and validation files"
|
|
764
|
+
});
|
|
765
|
+
}
|
|
766
|
+
});
|
|
767
|
+
var localBehaviorSpecFileSchema = specSnapshotSchema.extend({
|
|
768
|
+
id: z.string().uuid().optional(),
|
|
769
|
+
hyperparameters: fineTuneHyperparametersSchema.optional(),
|
|
770
|
+
dataset_prebuilt: datasetPrebuiltSchema.optional()
|
|
771
|
+
}).strict();
|
|
772
|
+
var evalExampleResultSchema = z.object({
|
|
773
|
+
prompt: z.string(),
|
|
774
|
+
expected: z.string(),
|
|
775
|
+
actual: z.string(),
|
|
776
|
+
passed: z.boolean(),
|
|
777
|
+
score: z.number().min(0).max(1),
|
|
778
|
+
reasoning: z.string().nullable(),
|
|
779
|
+
latency_ms: z.number().int().nonnegative(),
|
|
780
|
+
scored_by: z.enum(["exact_match", "json_fields", "heuristic"]).optional()
|
|
781
|
+
}).strict();
|
|
782
|
+
var jsonFieldMetricsSchema = z.object({
|
|
783
|
+
fields: z.array(z.string()),
|
|
784
|
+
valid_json_count: z.number().int().nonnegative(),
|
|
785
|
+
valid_json_rate: z.number().min(0).max(1),
|
|
786
|
+
schema_match_count: z.number().int().nonnegative(),
|
|
787
|
+
schema_match_rate: z.number().min(0).max(1),
|
|
788
|
+
all_fields_match_count: z.number().int().nonnegative(),
|
|
789
|
+
all_fields_match_rate: z.number().min(0).max(1),
|
|
790
|
+
field_accuracy: z.record(z.string(), z.object({
|
|
791
|
+
correct: z.number().int().nonnegative(),
|
|
792
|
+
total: z.number().int().nonnegative(),
|
|
793
|
+
accuracy: z.number().min(0).max(1)
|
|
794
|
+
}).strict())
|
|
795
|
+
}).strict();
|
|
796
|
+
var evalSplitSchema = z.enum([
|
|
797
|
+
"spec_examples",
|
|
798
|
+
"spec_holdout",
|
|
799
|
+
"prebuilt_test",
|
|
800
|
+
"prebuilt_validation",
|
|
801
|
+
"general_regression"
|
|
802
|
+
]);
|
|
803
|
+
var evalReportSchema = z.object({
|
|
804
|
+
kind: z.enum(["baseline", "candidate"]),
|
|
805
|
+
model_id: z.string(),
|
|
806
|
+
total: z.number().int().nonnegative(),
|
|
807
|
+
eval_examples_total: z.number().int().nonnegative(),
|
|
808
|
+
eval_examples_used: z.number().int().nonnegative(),
|
|
809
|
+
eval_truncated: z.boolean(),
|
|
810
|
+
eval_split: evalSplitSchema.optional(),
|
|
811
|
+
eval_sample_seed: z.number().int().nullable().optional(),
|
|
812
|
+
avg_score: z.number().min(0).max(1),
|
|
813
|
+
pass_rate: z.number().min(0).max(1),
|
|
814
|
+
exact_match_rate: z.number().min(0).max(1),
|
|
815
|
+
avg_token_f1: z.number().min(0).max(1).optional(),
|
|
816
|
+
avg_latency_ms: z.number().int().nonnegative(),
|
|
817
|
+
cached: z.boolean().optional(),
|
|
818
|
+
cache_key: z.string().optional(),
|
|
819
|
+
results: z.array(evalExampleResultSchema),
|
|
820
|
+
artifact_uri: z.string(),
|
|
821
|
+
scoring_method: z.enum(["heuristic", "exact_match", "json_fields"]),
|
|
822
|
+
inference_provider: z.enum(["none", "transformers"]).optional(),
|
|
823
|
+
scoring_mode: z.enum(["exact_match", "json_fields"]).optional(),
|
|
824
|
+
json_field_metrics: jsonFieldMetricsSchema.optional(),
|
|
825
|
+
generation_config: z.record(z.string(), z.unknown()).optional(),
|
|
826
|
+
log_uri: z.string().optional()
|
|
827
|
+
}).strict();
|
|
828
|
+
var comparisonReportSchema = z.object({
|
|
829
|
+
avg_score_delta: z.number(),
|
|
830
|
+
pass_rate_delta: z.number(),
|
|
831
|
+
exact_match_rate_delta: z.number(),
|
|
832
|
+
token_f1_delta: z.number().optional(),
|
|
833
|
+
regressions: z.number().int().nonnegative(),
|
|
834
|
+
improvements: z.number().int().nonnegative(),
|
|
835
|
+
regressed_examples: z.array(z.object({
|
|
836
|
+
prompt: z.string(),
|
|
837
|
+
old_score: z.number(),
|
|
838
|
+
new_score: z.number()
|
|
839
|
+
}).strict())
|
|
840
|
+
}).strict();
|
|
841
|
+
var trainingReportSchema = z.object({
|
|
842
|
+
provider: z.literal("local-uv"),
|
|
843
|
+
training_job_name: z.string(),
|
|
844
|
+
model_artifact_uri: z.string().optional(),
|
|
845
|
+
base_model_artifact_uri: z.string().optional(),
|
|
846
|
+
metrics: z.record(z.string(), z.unknown()).nullable(),
|
|
847
|
+
exit_code: z.number().int().nullable(),
|
|
848
|
+
log_uri: z.string(),
|
|
849
|
+
command: z.array(z.string()).optional()
|
|
850
|
+
}).strict();
|
|
851
|
+
var runReportSchema = z.object({
|
|
852
|
+
run_id: z.string().uuid(),
|
|
853
|
+
behavior_spec_id: z.string().uuid(),
|
|
854
|
+
user_id: z.string(),
|
|
855
|
+
run_number: z.number().int().min(1),
|
|
856
|
+
base_model: z.string(),
|
|
857
|
+
fine_tuned_model_id: z.string(),
|
|
858
|
+
status: z.enum(["completed", "failed"]),
|
|
859
|
+
baseline: evalReportSchema,
|
|
860
|
+
candidate: evalReportSchema,
|
|
861
|
+
comparison: comparisonReportSchema,
|
|
862
|
+
general_regression: z.object({
|
|
863
|
+
dataset_uri: z.string(),
|
|
864
|
+
dataset_sha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
865
|
+
baseline: evalReportSchema,
|
|
866
|
+
candidate: evalReportSchema,
|
|
867
|
+
comparison: comparisonReportSchema,
|
|
868
|
+
policy: z.object({
|
|
869
|
+
max_score_drop: z.number().min(0).max(1),
|
|
870
|
+
max_pass_rate_drop: z.number().min(0).max(1)
|
|
871
|
+
}).strict(),
|
|
872
|
+
passed: z.boolean(),
|
|
873
|
+
failures: z.array(z.string())
|
|
874
|
+
}).strict().optional(),
|
|
875
|
+
training: trainingReportSchema,
|
|
876
|
+
artifact_uris: z.object({
|
|
877
|
+
dataset: z.string(),
|
|
878
|
+
baseline_eval: z.string(),
|
|
879
|
+
candidate_eval: z.string(),
|
|
880
|
+
general_baseline_eval: z.string().optional(),
|
|
881
|
+
general_candidate_eval: z.string().optional(),
|
|
882
|
+
report: z.string()
|
|
883
|
+
}).strict(),
|
|
884
|
+
run_metadata: z.object({
|
|
885
|
+
base_model: z.string(),
|
|
886
|
+
fine_tuned_model_id: z.string(),
|
|
887
|
+
dataset_prebuilt: z.boolean(),
|
|
888
|
+
dataset_format: datasetFormatSchema.nullable().optional(),
|
|
889
|
+
dataset_uri: z.string(),
|
|
890
|
+
spec_example_count: z.number().int().nonnegative(),
|
|
891
|
+
training_example_count: z.number().int().nonnegative().nullable(),
|
|
892
|
+
eval_examples_total: z.number().int().nonnegative(),
|
|
893
|
+
eval_examples_used: z.number().int().nonnegative(),
|
|
894
|
+
eval_split: evalSplitSchema.optional(),
|
|
895
|
+
eval_sample_seed: z.number().int().nullable().optional(),
|
|
896
|
+
started_at: z.string(),
|
|
897
|
+
completed_at: z.string(),
|
|
898
|
+
elapsed_ms: z.number().int().nonnegative(),
|
|
899
|
+
elapsed_seconds: z.number().nonnegative()
|
|
900
|
+
}).strict(),
|
|
901
|
+
created_at: z.string()
|
|
902
|
+
}).strict();
|
|
903
|
+
var inferenceConfigSchema = z.object({
|
|
904
|
+
maxNewTokens: z.number().int().min(1).default(256),
|
|
905
|
+
temperature: z.number().min(0).default(0),
|
|
906
|
+
topP: z.number().min(0).max(1).default(1),
|
|
907
|
+
device: z.enum(["cuda", "cpu"]).default("cuda")
|
|
908
|
+
}).strict();
|
|
909
|
+
var scoringConfigSchema = z.object({
|
|
910
|
+
mode: z.enum(["exact_match", "json_fields"]).default("exact_match"),
|
|
911
|
+
fields: z.array(z.string().min(1)).optional()
|
|
912
|
+
}).strict();
|
|
913
|
+
var evaluationConfigSchema = z.object({
|
|
914
|
+
inference: inferenceConfigSchema.default({
|
|
915
|
+
maxNewTokens: 256,
|
|
916
|
+
temperature: 0,
|
|
917
|
+
topP: 1,
|
|
918
|
+
device: "cuda"
|
|
919
|
+
}),
|
|
920
|
+
scoring: scoringConfigSchema.default({ mode: "exact_match" }),
|
|
921
|
+
timeoutMs: z.number().int().min(100).default(18e5),
|
|
922
|
+
maxExamples: z.number().int().min(1).optional(),
|
|
923
|
+
sampleSeed: z.number().int().optional(),
|
|
924
|
+
baselineCache: z.boolean().default(true),
|
|
925
|
+
generalRegression: z.object({
|
|
926
|
+
dataset: z.string().min(1),
|
|
927
|
+
systemPrompt: z.string().optional(),
|
|
928
|
+
maxScoreDrop: z.number().min(0).max(1).default(0.03),
|
|
929
|
+
maxPassRateDrop: z.number().min(0).max(1).default(0.05)
|
|
930
|
+
}).strict().optional()
|
|
931
|
+
}).strict();
|
|
932
|
+
var localRunnerConfigSchema = z.object({
|
|
933
|
+
storeRoot: z.string().optional(),
|
|
934
|
+
artifactRoot: z.string().default(".tt-local/artifacts"),
|
|
935
|
+
dryRun: z.boolean().default(false),
|
|
936
|
+
paths: z.object({
|
|
937
|
+
baseModel: z.string().optional(),
|
|
938
|
+
modelCache: z.string().optional()
|
|
939
|
+
}).strict().default({}),
|
|
940
|
+
evaluation: evaluationConfigSchema.default({
|
|
941
|
+
inference: {
|
|
942
|
+
maxNewTokens: 256,
|
|
943
|
+
temperature: 0,
|
|
944
|
+
topP: 1,
|
|
945
|
+
device: "cuda"
|
|
946
|
+
},
|
|
947
|
+
scoring: { mode: "exact_match" },
|
|
948
|
+
timeoutMs: 18e5,
|
|
949
|
+
baselineCache: true
|
|
950
|
+
})
|
|
951
|
+
}).strict();
|
|
952
|
+
|
|
953
|
+
// src/local-runtime/local-project.ts
|
|
954
|
+
import { createHash as createHash2, randomUUID as randomUUID2 } from "crypto";
|
|
955
|
+
import { readFile as readFile2, stat as stat2, writeFile as writeFile2 } from "fs/promises";
|
|
956
|
+
import { dirname as dirname2, isAbsolute as isAbsolute2, resolve as resolve3 } from "path";
|
|
957
|
+
import { homedir } from "os";
|
|
958
|
+
import { mkdir as mkdir2 } from "fs/promises";
|
|
959
|
+
var DEFAULT_LOCAL_SPEC_PATH = "tunedtensor.json";
|
|
960
|
+
function resolveLocalReference(value, baseDirectory) {
|
|
961
|
+
if (typeof value !== "string" || !value || isAbsolute2(value) || /^[a-z][a-z0-9+.-]*:/i.test(value)) {
|
|
962
|
+
return value;
|
|
963
|
+
}
|
|
964
|
+
if (value === "~") return homedir();
|
|
965
|
+
if (value.startsWith("~/")) return resolve3(homedir(), value.slice(2));
|
|
966
|
+
return resolve3(baseDirectory, value);
|
|
967
|
+
}
|
|
968
|
+
function resolveLocalRunInputPaths(raw, inputPath) {
|
|
969
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return raw;
|
|
970
|
+
const baseDirectory = dirname2(resolve3(inputPath));
|
|
971
|
+
const value = structuredClone(raw);
|
|
972
|
+
const dataset = value.dataset_prebuilt;
|
|
973
|
+
if (dataset && typeof dataset === "object" && !Array.isArray(dataset)) {
|
|
974
|
+
const fields = dataset;
|
|
975
|
+
for (const key of ["training", "validation", "test"]) {
|
|
976
|
+
if (fields[key] !== void 0) fields[key] = resolveLocalReference(fields[key], baseDirectory);
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
return value;
|
|
980
|
+
}
|
|
981
|
+
async function exists(path) {
|
|
982
|
+
try {
|
|
983
|
+
await stat2(path);
|
|
984
|
+
return true;
|
|
985
|
+
} catch {
|
|
986
|
+
return false;
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
async function initLocalSpecFile(args) {
|
|
990
|
+
if (!args.force && await exists(args.outputPath)) {
|
|
991
|
+
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
992
|
+
}
|
|
993
|
+
const spec = localBehaviorSpecFileSchema.parse({
|
|
994
|
+
id: randomUUID2(),
|
|
995
|
+
name: args.name,
|
|
996
|
+
description: "",
|
|
997
|
+
system_prompt: "Describe the behavior this local model should learn.",
|
|
998
|
+
guidelines: [
|
|
999
|
+
"Return concise, task-specific answers."
|
|
1000
|
+
],
|
|
1001
|
+
constraints: [],
|
|
1002
|
+
base_model: args.baseModel,
|
|
1003
|
+
examples: [
|
|
1004
|
+
{
|
|
1005
|
+
input: "Replace this with representative training input.",
|
|
1006
|
+
output: "Replace this with the expected training output."
|
|
1007
|
+
},
|
|
1008
|
+
{
|
|
1009
|
+
input: "Replace this with a different validation input.",
|
|
1010
|
+
output: "Replace this with the expected validation output."
|
|
1011
|
+
}
|
|
1012
|
+
],
|
|
1013
|
+
hyperparameters: {
|
|
1014
|
+
n_epochs: 1
|
|
1015
|
+
}
|
|
1016
|
+
});
|
|
1017
|
+
await mkdir2(dirname2(args.outputPath), { recursive: true });
|
|
1018
|
+
await writeFile2(args.outputPath, `${JSON.stringify(spec, null, 2)}
|
|
1019
|
+
`, "utf8");
|
|
1020
|
+
return spec;
|
|
1021
|
+
}
|
|
1022
|
+
async function initLocalRunnerConfigFile(args) {
|
|
1023
|
+
if (!args.force && await exists(args.outputPath)) {
|
|
1024
|
+
throw new Error(`Refusing to overwrite existing file: ${args.outputPath}`);
|
|
1025
|
+
}
|
|
1026
|
+
const config = {
|
|
1027
|
+
artifactRoot: ".tt-local/artifacts",
|
|
1028
|
+
storeRoot: ".tt-local/store",
|
|
1029
|
+
evaluation: {
|
|
1030
|
+
inference: {
|
|
1031
|
+
device: "cuda"
|
|
1032
|
+
},
|
|
1033
|
+
scoring: {
|
|
1034
|
+
mode: "exact_match"
|
|
1035
|
+
},
|
|
1036
|
+
timeoutMs: 18e5
|
|
1037
|
+
}
|
|
1038
|
+
};
|
|
1039
|
+
await mkdir2(dirname2(args.outputPath), { recursive: true });
|
|
1040
|
+
await writeFile2(args.outputPath, `${JSON.stringify(config, null, 2)}
|
|
1041
|
+
`, "utf8");
|
|
1042
|
+
return config;
|
|
1043
|
+
}
|
|
1044
|
+
function generatedPlaceholderIssues(request) {
|
|
1045
|
+
const issues = [];
|
|
1046
|
+
if (/describe the behavior this local model should learn/i.test(request.spec_snapshot.system_prompt)) {
|
|
1047
|
+
issues.push("system_prompt still contains the generated placeholder");
|
|
1048
|
+
}
|
|
1049
|
+
request.spec_snapshot.examples.forEach((example, index) => {
|
|
1050
|
+
if (/replace this with/i.test(example.input) || /replace this with/i.test(example.output)) {
|
|
1051
|
+
issues.push(`examples[${index}] still contains generated placeholder text`);
|
|
1052
|
+
}
|
|
1053
|
+
});
|
|
1054
|
+
return issues;
|
|
1055
|
+
}
|
|
1056
|
+
function assertLocalRunInputReady(request) {
|
|
1057
|
+
const issues = generatedPlaceholderIssues(request);
|
|
1058
|
+
if (issues.length > 0) {
|
|
1059
|
+
throw new Error(`Edit the generated behavior spec before training: ${issues.join("; ")}.`);
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
function runRequestFromLocalSpec(spec, options = {}) {
|
|
1063
|
+
const {
|
|
1064
|
+
id,
|
|
1065
|
+
hyperparameters,
|
|
1066
|
+
dataset_prebuilt: datasetPrebuilt,
|
|
1067
|
+
...specSnapshot
|
|
1068
|
+
} = spec;
|
|
1069
|
+
const behaviorSpecId = id ?? deterministicBehaviorSpecId(specSnapshot);
|
|
1070
|
+
return fineTuneRunRequestSchema.parse({
|
|
1071
|
+
run_id: options.runId ?? randomUUID2(),
|
|
1072
|
+
user_id: "local-user",
|
|
1073
|
+
behavior_spec_id: behaviorSpecId,
|
|
1074
|
+
run_number: 1,
|
|
1075
|
+
spec_snapshot: specSnapshot,
|
|
1076
|
+
hyperparameters,
|
|
1077
|
+
dataset_prebuilt: datasetPrebuilt
|
|
1078
|
+
});
|
|
1079
|
+
}
|
|
1080
|
+
function deterministicBehaviorSpecId(spec) {
|
|
1081
|
+
const namespace = Buffer.from("9f783f01340457bea061942004259253", "hex");
|
|
1082
|
+
const digest = createHash2("sha1").update(namespace).update(JSON.stringify(spec)).digest().subarray(0, 16);
|
|
1083
|
+
digest[6] = digest[6] & 15 | 80;
|
|
1084
|
+
digest[8] = digest[8] & 63 | 128;
|
|
1085
|
+
const hex = digest.toString("hex");
|
|
1086
|
+
return [
|
|
1087
|
+
hex.slice(0, 8),
|
|
1088
|
+
hex.slice(8, 12),
|
|
1089
|
+
hex.slice(12, 16),
|
|
1090
|
+
hex.slice(16, 20),
|
|
1091
|
+
hex.slice(20)
|
|
1092
|
+
].join("-");
|
|
1093
|
+
}
|
|
1094
|
+
async function loadLocalRunInput(path, options = {}) {
|
|
1095
|
+
const raw = resolveLocalRunInputPaths(JSON.parse(await readFile2(path, "utf8")), path);
|
|
1096
|
+
const request = fineTuneRunRequestSchema.safeParse(raw);
|
|
1097
|
+
if (request.success) {
|
|
1098
|
+
return { kind: "request", path, request: request.data };
|
|
1099
|
+
}
|
|
1100
|
+
const spec = localBehaviorSpecFileSchema.parse(raw);
|
|
1101
|
+
return {
|
|
1102
|
+
kind: "spec",
|
|
1103
|
+
path,
|
|
1104
|
+
spec,
|
|
1105
|
+
request: runRequestFromLocalSpec(spec, options)
|
|
1106
|
+
};
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
// src/local-runtime/dataset.ts
|
|
1110
|
+
import { readFile as readFile3 } from "fs/promises";
|
|
1111
|
+
async function loadNormalizedChatJsonl(path) {
|
|
1112
|
+
const text = await readFile3(path, "utf8");
|
|
1113
|
+
const rows = [];
|
|
1114
|
+
const jsonLines = [];
|
|
1115
|
+
for (const [index, line] of text.split(/\r?\n/).entries()) {
|
|
1116
|
+
if (!line.trim()) continue;
|
|
1117
|
+
let value;
|
|
1118
|
+
try {
|
|
1119
|
+
value = JSON.parse(line);
|
|
1120
|
+
} catch (error) {
|
|
1121
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: malformed JSON`, { cause: error });
|
|
1122
|
+
}
|
|
1123
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
1124
|
+
throw new Error(`Invalid chat JSONL row ${index + 1}: expected an object`);
|
|
1125
|
+
}
|
|
1126
|
+
const messages = value.messages;
|
|
1127
|
+
if (!Array.isArray(messages) || ![2, 3].includes(messages.length) || messages.some(
|
|
1128
|
+
(message) => !message || typeof message !== "object" || Array.isArray(message) || !["system", "user", "assistant"].includes(String(message.role)) || typeof message.content !== "string"
|
|
1129
|
+
)) {
|
|
1130
|
+
throw new Error(
|
|
1131
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one user message, and one assistant answer`
|
|
1132
|
+
);
|
|
1133
|
+
}
|
|
1134
|
+
const row = { messages };
|
|
1135
|
+
const offset = row.messages.length === 3 ? 1 : 0;
|
|
1136
|
+
if (offset === 1 && row.messages[0]?.role !== "system" || row.messages[offset]?.role !== "user" || !row.messages[offset]?.content.trim() || row.messages[offset + 1]?.role !== "assistant" || !row.messages[offset + 1]?.content.trim()) {
|
|
1137
|
+
throw new Error(
|
|
1138
|
+
`Invalid chat JSONL row ${index + 1}: expected an optional system message, one non-empty user message, and one non-empty assistant answer`
|
|
1139
|
+
);
|
|
1140
|
+
}
|
|
1141
|
+
rows.push(row);
|
|
1142
|
+
jsonLines.push(JSON.stringify(row));
|
|
1143
|
+
}
|
|
1144
|
+
if (rows.length === 0) throw new Error(`Chat JSONL contains no examples: ${path}`);
|
|
1145
|
+
return {
|
|
1146
|
+
rows,
|
|
1147
|
+
jsonl: jsonLines.join("\n")
|
|
1148
|
+
};
|
|
1149
|
+
}
|
|
1150
|
+
async function normalizeChatJsonlForRelocation(path) {
|
|
1151
|
+
return (await loadNormalizedChatJsonl(path)).jsonl;
|
|
1152
|
+
}
|
|
1153
|
+
function buildSystemMessage(spec) {
|
|
1154
|
+
const parts = [];
|
|
1155
|
+
if (spec.system_prompt.trim()) parts.push(spec.system_prompt.trim());
|
|
1156
|
+
if (spec.guidelines.length > 0) {
|
|
1157
|
+
parts.push(`Guidelines:
|
|
1158
|
+
${spec.guidelines.map((guideline) => `- ${guideline}`).join("\n")}`);
|
|
1159
|
+
}
|
|
1160
|
+
if (spec.constraints.length > 0) {
|
|
1161
|
+
parts.push(`Constraints:
|
|
1162
|
+
${spec.constraints.map((constraint) => `- ${constraint}`).join("\n")}`);
|
|
1163
|
+
}
|
|
1164
|
+
return parts.join("\n\n") || "Follow the demonstrated behavior.";
|
|
1165
|
+
}
|
|
1166
|
+
function exampleToChatRow(spec, example) {
|
|
1167
|
+
const system = buildSystemMessage(spec);
|
|
1168
|
+
return {
|
|
1169
|
+
messages: [
|
|
1170
|
+
{ role: "system", content: system },
|
|
1171
|
+
{ role: "user", content: example.input },
|
|
1172
|
+
{ role: "assistant", content: example.output }
|
|
1173
|
+
]
|
|
1174
|
+
};
|
|
1175
|
+
}
|
|
1176
|
+
function compileSpecToJsonl(spec) {
|
|
1177
|
+
return spec.examples.map((example) => JSON.stringify(exampleToChatRow(spec, example))).join("\n");
|
|
1178
|
+
}
|
|
1179
|
+
function examplesFromSpec(spec) {
|
|
1180
|
+
return spec.examples;
|
|
1181
|
+
}
|
|
1182
|
+
async function examplesFromChatJsonl(path) {
|
|
1183
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
1184
|
+
const examples = [];
|
|
1185
|
+
for (const [index, row] of normalized.rows.entries()) {
|
|
1186
|
+
const user = row.messages.at(-2);
|
|
1187
|
+
const assistant = row.messages.at(-1);
|
|
1188
|
+
examples.push({
|
|
1189
|
+
input: user.content,
|
|
1190
|
+
output: assistant.content
|
|
1191
|
+
});
|
|
1192
|
+
}
|
|
1193
|
+
return examples;
|
|
1194
|
+
}
|
|
1195
|
+
async function evaluationSuiteFromChatJsonl(path, configuredSystem) {
|
|
1196
|
+
const normalized = await loadNormalizedChatJsonl(path);
|
|
1197
|
+
const inputIdentities = /* @__PURE__ */ new Set();
|
|
1198
|
+
for (const row of normalized.rows) {
|
|
1199
|
+
const input = row.messages.at(-2).content;
|
|
1200
|
+
const identity = input.trim().replace(/\s+/g, " ").toLowerCase();
|
|
1201
|
+
if (inputIdentities.has(identity)) {
|
|
1202
|
+
throw new Error(
|
|
1203
|
+
"General regression data contains duplicate inputs; evaluation prompts must be unique."
|
|
1204
|
+
);
|
|
1205
|
+
}
|
|
1206
|
+
inputIdentities.add(identity);
|
|
1207
|
+
}
|
|
1208
|
+
const systems = new Set(
|
|
1209
|
+
normalized.rows.map((row) => row.messages.find((message) => message.role === "system")?.content).filter((value) => value !== void 0)
|
|
1210
|
+
);
|
|
1211
|
+
if (systems.size > 1) {
|
|
1212
|
+
throw new Error("General regression rows must use one consistent system message.");
|
|
1213
|
+
}
|
|
1214
|
+
const embeddedSystem = [...systems][0];
|
|
1215
|
+
if (configuredSystem !== void 0 && embeddedSystem !== void 0 && configuredSystem !== embeddedSystem) {
|
|
1216
|
+
throw new Error(
|
|
1217
|
+
"evaluation.generalRegression.systemPrompt must match the system message embedded in the dataset."
|
|
1218
|
+
);
|
|
1219
|
+
}
|
|
1220
|
+
return {
|
|
1221
|
+
examples: normalized.rows.map((row) => ({
|
|
1222
|
+
input: row.messages.at(-2).content,
|
|
1223
|
+
output: row.messages.at(-1).content
|
|
1224
|
+
})),
|
|
1225
|
+
system: configuredSystem ?? embeddedSystem ?? "You are a helpful assistant."
|
|
1226
|
+
};
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
// src/local-runtime/run-reporter.ts
|
|
1230
|
+
function reportInBackground(action) {
|
|
1231
|
+
try {
|
|
1232
|
+
void Promise.resolve(action()).catch(() => void 0);
|
|
1233
|
+
} catch {
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
function sanitizeLogLine(line) {
|
|
1237
|
+
return line.replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|secret|password)=)[^\s]+/gi, "$1[redacted]");
|
|
1238
|
+
}
|
|
1239
|
+
function forwardStreamLines(stream, onLine) {
|
|
1240
|
+
let buffered = "";
|
|
1241
|
+
stream.setEncoding("utf8");
|
|
1242
|
+
stream.on("data", (chunk) => {
|
|
1243
|
+
buffered += chunk;
|
|
1244
|
+
const lines = buffered.split(/\r\n|\n|\r/);
|
|
1245
|
+
buffered = lines.pop() ?? "";
|
|
1246
|
+
for (const line of lines) {
|
|
1247
|
+
if (line.trim()) onLine(sanitizeLogLine(line));
|
|
1248
|
+
}
|
|
1249
|
+
});
|
|
1250
|
+
stream.on("end", () => {
|
|
1251
|
+
if (buffered.trim()) onLine(sanitizeLogLine(buffered));
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
// src/local-runtime/package-root.ts
|
|
1256
|
+
import { existsSync } from "fs";
|
|
1257
|
+
import { dirname as dirname3, resolve as resolve4 } from "path";
|
|
1258
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
1259
|
+
function localRuntimePackageRoot(moduleUrl) {
|
|
1260
|
+
const moduleDirectory = dirname3(fileURLToPath2(moduleUrl));
|
|
1261
|
+
const candidates = [
|
|
1262
|
+
dirname3(moduleDirectory),
|
|
1263
|
+
dirname3(dirname3(moduleDirectory))
|
|
1264
|
+
];
|
|
1265
|
+
const root = candidates.find(
|
|
1266
|
+
(candidate) => existsSync(resolve4(candidate, "package.json")) && existsSync(resolve4(candidate, "training/local-runner"))
|
|
1267
|
+
);
|
|
1268
|
+
if (!root) {
|
|
1269
|
+
throw new Error("Unable to locate the bundled Tuned Tensor local runtime assets.");
|
|
1270
|
+
}
|
|
1271
|
+
return root;
|
|
1272
|
+
}
|
|
1273
|
+
|
|
1274
|
+
// src/local-runtime/store.ts
|
|
1275
|
+
import { appendFile, copyFile as copyFile2, mkdir as mkdir3, readdir as readdir3, readFile as readFile4, rename as rename2, rm, stat as stat3, writeFile as writeFile3 } from "fs/promises";
|
|
1276
|
+
import { homedir as homedir2 } from "os";
|
|
1277
|
+
import { dirname as dirname4, join as join2, resolve as resolve5 } from "path";
|
|
1278
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
1279
|
+
function isTerminalRunState(state) {
|
|
1280
|
+
return state.status === "stage_completed" || state.status === "completed" || state.status === "failed" || state.status === "cancelled" && state.current_stage === "cancelled" && Boolean(state.completed_at);
|
|
1281
|
+
}
|
|
1282
|
+
function defaultLocalHome() {
|
|
1283
|
+
return resolve5(process.env.TT_LOCAL_HOME ?? join2(homedir2(), ".tuned-tensor-local"));
|
|
1284
|
+
}
|
|
1285
|
+
function localStorePaths(root) {
|
|
1286
|
+
return {
|
|
1287
|
+
root,
|
|
1288
|
+
specsDir: join2(root, "specs"),
|
|
1289
|
+
runsDir: join2(root, "runs"),
|
|
1290
|
+
modelsDir: join2(root, "models"),
|
|
1291
|
+
datasetsDir: join2(root, "datasets")
|
|
1292
|
+
};
|
|
1293
|
+
}
|
|
1294
|
+
async function exists2(path) {
|
|
1295
|
+
try {
|
|
1296
|
+
await stat3(path);
|
|
1297
|
+
return true;
|
|
1298
|
+
} catch {
|
|
1299
|
+
return false;
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
async function writeJsonAtomic2(path, value) {
|
|
1303
|
+
await mkdir3(dirname4(path), { recursive: true });
|
|
1304
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.${randomUUID3()}.tmp`;
|
|
1305
|
+
await writeFile3(tmp, `${JSON.stringify(value, null, 2)}
|
|
1306
|
+
`, "utf8");
|
|
1307
|
+
await rename2(tmp, path);
|
|
1308
|
+
}
|
|
1309
|
+
async function readJson2(path) {
|
|
1310
|
+
return JSON.parse(await readFile4(path, "utf8"));
|
|
1311
|
+
}
|
|
1312
|
+
async function appendJsonl(path, value) {
|
|
1313
|
+
await mkdir3(dirname4(path), { recursive: true });
|
|
1314
|
+
await appendFile(path, `${JSON.stringify(value)}
|
|
1315
|
+
`, "utf8");
|
|
1316
|
+
}
|
|
1317
|
+
async function readJsonl(path) {
|
|
1318
|
+
if (!await exists2(path)) return [];
|
|
1319
|
+
return (await readFile4(path, "utf8")).split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
|
|
1320
|
+
}
|
|
1321
|
+
async function copyIfExists(from, to) {
|
|
1322
|
+
if (!await exists2(from)) return;
|
|
1323
|
+
await mkdir3(dirname4(to), { recursive: true });
|
|
1324
|
+
await copyFile2(from, to);
|
|
1325
|
+
}
|
|
1326
|
+
async function childDirectoryNames(path) {
|
|
1327
|
+
try {
|
|
1328
|
+
const entries = await readdir3(path, { withFileTypes: true });
|
|
1329
|
+
return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
1330
|
+
} catch (error) {
|
|
1331
|
+
if (error.code === "ENOENT") return [];
|
|
1332
|
+
throw error;
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
function compareIds(left, right) {
|
|
1336
|
+
if (left.id === right.id) return 0;
|
|
1337
|
+
return left.id < right.id ? -1 : 1;
|
|
1338
|
+
}
|
|
1339
|
+
function compareUpdatedAt(left, right) {
|
|
1340
|
+
if (left.updated_at !== right.updated_at) return left.updated_at > right.updated_at ? -1 : 1;
|
|
1341
|
+
return compareIds(left, right);
|
|
1342
|
+
}
|
|
1343
|
+
function compareCreatedAt(left, right) {
|
|
1344
|
+
if (left.created_at !== right.created_at) return left.created_at > right.created_at ? -1 : 1;
|
|
1345
|
+
return compareIds(left, right);
|
|
1346
|
+
}
|
|
1347
|
+
function findByIdOrPrefix(records, id) {
|
|
1348
|
+
return records.find((record) => record.id === id) ?? records.find((record) => record.id.startsWith(id));
|
|
1349
|
+
}
|
|
1350
|
+
function createLocalStore(root = defaultLocalHome()) {
|
|
1351
|
+
const resolvedRoot = resolve5(root);
|
|
1352
|
+
const paths = localStorePaths(resolvedRoot);
|
|
1353
|
+
const runDir = (id) => join2(paths.runsDir, id);
|
|
1354
|
+
const runStatePath = (id) => join2(runDir(id), "state.json");
|
|
1355
|
+
const runEventsPath = (id) => join2(runDir(id), "progress.jsonl");
|
|
1356
|
+
const runRequestPath = (id) => join2(runDir(id), "request.json");
|
|
1357
|
+
const runReportPath = (id) => join2(runDir(id), "run-report.json");
|
|
1358
|
+
const cancellationPath = (id) => join2(runDir(id), "cancel.requested");
|
|
1359
|
+
const specDir = (id) => join2(paths.specsDir, id);
|
|
1360
|
+
const specPath = (id) => join2(specDir(id), "spec.json");
|
|
1361
|
+
const modelPath = (id) => join2(paths.modelsDir, id, "model.json");
|
|
1362
|
+
async function ensure() {
|
|
1363
|
+
await Promise.all([
|
|
1364
|
+
mkdir3(paths.specsDir, { recursive: true }),
|
|
1365
|
+
mkdir3(paths.runsDir, { recursive: true }),
|
|
1366
|
+
mkdir3(paths.modelsDir, { recursive: true }),
|
|
1367
|
+
mkdir3(paths.datasetsDir, { recursive: true })
|
|
1368
|
+
]);
|
|
1369
|
+
}
|
|
1370
|
+
async function writeRunState(state) {
|
|
1371
|
+
await writeJsonAtomic2(runStatePath(state.id), state);
|
|
1372
|
+
return state;
|
|
1373
|
+
}
|
|
1374
|
+
async function readSpecRecord(id) {
|
|
1375
|
+
const path = specPath(id);
|
|
1376
|
+
if (!await exists2(path)) return null;
|
|
1377
|
+
const [spec, fileStats, directoryStats] = await Promise.all([
|
|
1378
|
+
readJson2(path),
|
|
1379
|
+
stat3(path),
|
|
1380
|
+
stat3(specDir(id))
|
|
1381
|
+
]);
|
|
1382
|
+
const createdAt = directoryStats.birthtimeMs > 0 ? directoryStats.birthtime : directoryStats.ctime;
|
|
1383
|
+
return {
|
|
1384
|
+
id,
|
|
1385
|
+
name: spec.name,
|
|
1386
|
+
base_model: spec.base_model,
|
|
1387
|
+
path,
|
|
1388
|
+
created_at: createdAt.toISOString(),
|
|
1389
|
+
updated_at: fileStats.mtime.toISOString()
|
|
1390
|
+
};
|
|
1391
|
+
}
|
|
1392
|
+
async function listSpecRecords() {
|
|
1393
|
+
await ensure();
|
|
1394
|
+
const records = [];
|
|
1395
|
+
for (const id of await childDirectoryNames(paths.specsDir)) {
|
|
1396
|
+
const record = await readSpecRecord(id);
|
|
1397
|
+
if (record) records.push(record);
|
|
1398
|
+
}
|
|
1399
|
+
return records.sort(compareUpdatedAt);
|
|
1400
|
+
}
|
|
1401
|
+
async function listRunRecords() {
|
|
1402
|
+
await ensure();
|
|
1403
|
+
const records = [];
|
|
1404
|
+
for (const id of await childDirectoryNames(paths.runsDir)) {
|
|
1405
|
+
const path = runStatePath(id);
|
|
1406
|
+
if (await exists2(path)) records.push(await readJson2(path));
|
|
1407
|
+
}
|
|
1408
|
+
return records.sort(compareUpdatedAt);
|
|
1409
|
+
}
|
|
1410
|
+
async function listModelRecords() {
|
|
1411
|
+
await ensure();
|
|
1412
|
+
const records = [];
|
|
1413
|
+
for (const id of await childDirectoryNames(paths.modelsDir)) {
|
|
1414
|
+
const path = modelPath(id);
|
|
1415
|
+
if (await exists2(path)) records.push(await readJson2(path));
|
|
1416
|
+
}
|
|
1417
|
+
return records.sort(compareCreatedAt);
|
|
1418
|
+
}
|
|
1419
|
+
async function importSpecRecord(specId, spec) {
|
|
1420
|
+
await ensure();
|
|
1421
|
+
await mkdir3(specDir(specId), { recursive: true });
|
|
1422
|
+
await writeJsonAtomic2(specPath(specId), spec);
|
|
1423
|
+
const record = await readSpecRecord(specId);
|
|
1424
|
+
if (!record) throw new Error(`Failed to persist spec: ${specId}`);
|
|
1425
|
+
return record;
|
|
1426
|
+
}
|
|
1427
|
+
async function appendRunEvent(state, event) {
|
|
1428
|
+
const row = {
|
|
1429
|
+
id: randomUUID3(),
|
|
1430
|
+
run_id: state.id,
|
|
1431
|
+
occurred_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1432
|
+
...event
|
|
1433
|
+
};
|
|
1434
|
+
await appendJsonl(runEventsPath(state.id), row);
|
|
1435
|
+
await appendJsonl(join2(state.artifact_dir, "progress.jsonl"), {
|
|
1436
|
+
at: row.occurred_at,
|
|
1437
|
+
stage: row.stage,
|
|
1438
|
+
status: row.status,
|
|
1439
|
+
message: row.message,
|
|
1440
|
+
...row.details ? { details: row.details } : {}
|
|
1441
|
+
});
|
|
1442
|
+
}
|
|
1443
|
+
async function finalizeCancellationState(runId) {
|
|
1444
|
+
const previous = await getRun(runId);
|
|
1445
|
+
if (previous.status === "cancelled" && previous.current_stage === "cancelled" && previous.completed_at) {
|
|
1446
|
+
return previous;
|
|
1447
|
+
}
|
|
1448
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1449
|
+
const cancelled = {
|
|
1450
|
+
...previous,
|
|
1451
|
+
status: "cancelled",
|
|
1452
|
+
current_stage: "cancelled",
|
|
1453
|
+
status_message: "Run cancelled.",
|
|
1454
|
+
error: void 0,
|
|
1455
|
+
completed_at: now,
|
|
1456
|
+
updated_at: now
|
|
1457
|
+
};
|
|
1458
|
+
await writeRunState(cancelled);
|
|
1459
|
+
await appendRunEvent(cancelled, { stage: "cancelled", status: "cancelled", message: "Run cancelled." });
|
|
1460
|
+
return cancelled;
|
|
1461
|
+
}
|
|
1462
|
+
async function preserveCancellationRequestState(runId) {
|
|
1463
|
+
const previous = await getRun(runId);
|
|
1464
|
+
if (isTerminalRunState(previous)) return previous;
|
|
1465
|
+
if (previous.status === "cancelled" && previous.current_stage === "cancel_requested") return previous;
|
|
1466
|
+
const requested = {
|
|
1467
|
+
...previous,
|
|
1468
|
+
status: "cancelled",
|
|
1469
|
+
current_stage: "cancel_requested",
|
|
1470
|
+
status_message: "Cancellation requested; waiting for the active worker to stop.",
|
|
1471
|
+
error: void 0,
|
|
1472
|
+
completed_at: void 0,
|
|
1473
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1474
|
+
};
|
|
1475
|
+
return writeRunState(requested);
|
|
1476
|
+
}
|
|
1477
|
+
async function resolveRunId(id) {
|
|
1478
|
+
await ensure();
|
|
1479
|
+
if (await exists2(runStatePath(id))) return id;
|
|
1480
|
+
const record = findByIdOrPrefix(await listRunRecords(), id);
|
|
1481
|
+
if (!record) throw new Error(`Run not found: ${id}`);
|
|
1482
|
+
return record.id;
|
|
1483
|
+
}
|
|
1484
|
+
async function getRun(id) {
|
|
1485
|
+
return readJson2(runStatePath(await resolveRunId(id)));
|
|
1486
|
+
}
|
|
1487
|
+
async function registerModelRecord(args) {
|
|
1488
|
+
if (!args.training.model_artifact_uri || args.training.metrics?.dry_run === true) return null;
|
|
1489
|
+
const previous = await getRun(args.request.run_id);
|
|
1490
|
+
const modelId = `local-${args.request.run_id}`;
|
|
1491
|
+
const existingModel = await exists2(modelPath(modelId)) ? await readJson2(modelPath(modelId)) : void 0;
|
|
1492
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1493
|
+
const model = {
|
|
1494
|
+
id: modelId,
|
|
1495
|
+
run_id: args.request.run_id,
|
|
1496
|
+
behavior_spec_id: args.request.behavior_spec_id,
|
|
1497
|
+
name: `${args.request.spec_snapshot.base_model} (${args.request.run_id.slice(0, 8)})`,
|
|
1498
|
+
provider: args.training.provider,
|
|
1499
|
+
base_model: args.request.spec_snapshot.base_model,
|
|
1500
|
+
artifact_uri: args.training.model_artifact_uri,
|
|
1501
|
+
artifact_dir: args.artifactDir,
|
|
1502
|
+
metrics: args.training.metrics,
|
|
1503
|
+
created_at: existingModel?.created_at ?? args.createdAt ?? now
|
|
1504
|
+
};
|
|
1505
|
+
await writeJsonAtomic2(modelPath(model.id), model);
|
|
1506
|
+
const alreadyRegistered = previous.model_id === modelId;
|
|
1507
|
+
const state = {
|
|
1508
|
+
...previous,
|
|
1509
|
+
model_id: modelId,
|
|
1510
|
+
updated_at: now
|
|
1511
|
+
};
|
|
1512
|
+
await writeRunState(state);
|
|
1513
|
+
if (!alreadyRegistered) {
|
|
1514
|
+
await appendRunEvent(state, {
|
|
1515
|
+
stage: "model_registered",
|
|
1516
|
+
status: "running",
|
|
1517
|
+
message: "Local model artifact registered.",
|
|
1518
|
+
details: { model_id: modelId, artifact_uri: model.artifact_uri }
|
|
1519
|
+
});
|
|
1520
|
+
}
|
|
1521
|
+
return model;
|
|
1522
|
+
}
|
|
1523
|
+
return {
|
|
1524
|
+
root: resolvedRoot,
|
|
1525
|
+
paths,
|
|
1526
|
+
ensure,
|
|
1527
|
+
importSpec: importSpecRecord,
|
|
1528
|
+
async listSpecs() {
|
|
1529
|
+
return listSpecRecords();
|
|
1530
|
+
},
|
|
1531
|
+
async getSpec(id) {
|
|
1532
|
+
const record = findByIdOrPrefix(await listSpecRecords(), id);
|
|
1533
|
+
if (!record) throw new Error(`Spec not found: ${id}`);
|
|
1534
|
+
return { ...record, spec: await readJson2(record.path) };
|
|
1535
|
+
},
|
|
1536
|
+
async startRun({ request, artifactDir }) {
|
|
1537
|
+
await ensure();
|
|
1538
|
+
await importSpecRecord(request.behavior_spec_id, request.spec_snapshot);
|
|
1539
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1540
|
+
const state = {
|
|
1541
|
+
id: request.run_id,
|
|
1542
|
+
behavior_spec_id: request.behavior_spec_id,
|
|
1543
|
+
user_id: request.user_id,
|
|
1544
|
+
run_number: request.run_number,
|
|
1545
|
+
status: "queued",
|
|
1546
|
+
current_stage: "queued",
|
|
1547
|
+
status_message: "Run queued.",
|
|
1548
|
+
artifact_dir: artifactDir,
|
|
1549
|
+
base_model: request.spec_snapshot.base_model,
|
|
1550
|
+
spec_name: request.spec_snapshot.name,
|
|
1551
|
+
created_at: now,
|
|
1552
|
+
updated_at: now,
|
|
1553
|
+
started_at: now
|
|
1554
|
+
};
|
|
1555
|
+
await writeJsonAtomic2(runRequestPath(request.run_id), request);
|
|
1556
|
+
await writeRunState(state);
|
|
1557
|
+
await appendRunEvent(state, { stage: "queued", status: "queued", message: "Run queued." });
|
|
1558
|
+
return state;
|
|
1559
|
+
},
|
|
1560
|
+
async syncRunRequest(request, artifactDir) {
|
|
1561
|
+
const previous = await getRun(request.run_id);
|
|
1562
|
+
if (previous.behavior_spec_id !== request.behavior_spec_id || previous.user_id !== request.user_id || previous.run_number !== request.run_number) {
|
|
1563
|
+
throw new Error(
|
|
1564
|
+
`Run ${request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
|
|
1565
|
+
);
|
|
1566
|
+
}
|
|
1567
|
+
await importSpecRecord(request.behavior_spec_id, request.spec_snapshot);
|
|
1568
|
+
await writeJsonAtomic2(runRequestPath(request.run_id), request);
|
|
1569
|
+
const state = {
|
|
1570
|
+
...previous,
|
|
1571
|
+
artifact_dir: artifactDir ?? previous.artifact_dir,
|
|
1572
|
+
base_model: request.spec_snapshot.base_model,
|
|
1573
|
+
spec_name: request.spec_snapshot.name,
|
|
1574
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1575
|
+
};
|
|
1576
|
+
await writeRunState(state);
|
|
1577
|
+
return state;
|
|
1578
|
+
},
|
|
1579
|
+
async updateRun({ runId, status, stage, message, details }) {
|
|
1580
|
+
const previous = await getRun(runId);
|
|
1581
|
+
if (status !== "cancelled" && await exists2(cancellationPath(previous.id))) {
|
|
1582
|
+
return preserveCancellationRequestState(previous.id);
|
|
1583
|
+
}
|
|
1584
|
+
const active = status !== "stage_completed" && status !== "completed" && status !== "failed" && status !== "cancelled";
|
|
1585
|
+
const successful = status === "stage_completed" || status === "completed";
|
|
1586
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1587
|
+
const state = {
|
|
1588
|
+
...previous,
|
|
1589
|
+
status,
|
|
1590
|
+
current_stage: stage,
|
|
1591
|
+
status_message: message,
|
|
1592
|
+
...active ? { error: void 0, completed_at: void 0 } : successful ? { error: void 0, completed_at: now } : {},
|
|
1593
|
+
updated_at: now
|
|
1594
|
+
};
|
|
1595
|
+
await writeRunState(state);
|
|
1596
|
+
if (status !== "cancelled" && await exists2(cancellationPath(previous.id))) {
|
|
1597
|
+
return preserveCancellationRequestState(previous.id);
|
|
1598
|
+
}
|
|
1599
|
+
await appendRunEvent(state, {
|
|
1600
|
+
stage,
|
|
1601
|
+
status: successful ? "completed" : status === "failed" ? "failed" : "running",
|
|
1602
|
+
message,
|
|
1603
|
+
details
|
|
1604
|
+
});
|
|
1605
|
+
return state;
|
|
1606
|
+
},
|
|
1607
|
+
async completeRun(report, artifactDir, reportPath) {
|
|
1608
|
+
if (await exists2(cancellationPath(report.run_id))) {
|
|
1609
|
+
return preserveCancellationRequestState(report.run_id);
|
|
1610
|
+
}
|
|
1611
|
+
const request = await readJson2(runRequestPath(report.run_id));
|
|
1612
|
+
const completedAt = report.created_at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
1613
|
+
const model = await registerModelRecord({
|
|
1614
|
+
request,
|
|
1615
|
+
training: report.training,
|
|
1616
|
+
artifactDir,
|
|
1617
|
+
createdAt: completedAt
|
|
1618
|
+
});
|
|
1619
|
+
if (!model) {
|
|
1620
|
+
await rm(join2(paths.modelsDir, `local-${report.run_id}`), { recursive: true, force: true });
|
|
1621
|
+
}
|
|
1622
|
+
const previous = await getRun(report.run_id);
|
|
1623
|
+
const state = {
|
|
1624
|
+
...previous,
|
|
1625
|
+
status: "completed",
|
|
1626
|
+
current_stage: "completed",
|
|
1627
|
+
status_message: "Run completed successfully.",
|
|
1628
|
+
report_path: reportPath,
|
|
1629
|
+
model_id: model?.id,
|
|
1630
|
+
error: void 0,
|
|
1631
|
+
completed_at: completedAt,
|
|
1632
|
+
updated_at: completedAt
|
|
1633
|
+
};
|
|
1634
|
+
await writeRunState(state);
|
|
1635
|
+
if (await exists2(cancellationPath(report.run_id))) {
|
|
1636
|
+
return preserveCancellationRequestState(report.run_id);
|
|
1637
|
+
}
|
|
1638
|
+
await copyIfExists(reportPath, runReportPath(report.run_id));
|
|
1639
|
+
await appendRunEvent(state, {
|
|
1640
|
+
stage: "completed",
|
|
1641
|
+
status: "completed",
|
|
1642
|
+
message: "Run completed successfully.",
|
|
1643
|
+
details: { report_path: reportPath, ...model ? { model_id: model.id } : {} }
|
|
1644
|
+
});
|
|
1645
|
+
return state;
|
|
1646
|
+
},
|
|
1647
|
+
registerModel: registerModelRecord,
|
|
1648
|
+
async invalidateRunOutputs(runId, options) {
|
|
1649
|
+
const previous = await getRun(runId);
|
|
1650
|
+
if (options.report) await rm(runReportPath(previous.id), { force: true });
|
|
1651
|
+
if (options.model) {
|
|
1652
|
+
await rm(join2(paths.modelsDir, `local-${previous.id}`), { recursive: true, force: true });
|
|
1653
|
+
}
|
|
1654
|
+
const state = {
|
|
1655
|
+
...previous,
|
|
1656
|
+
...options.report ? { report_path: void 0 } : {},
|
|
1657
|
+
...options.model ? { model_id: void 0 } : {},
|
|
1658
|
+
error: void 0,
|
|
1659
|
+
completed_at: void 0,
|
|
1660
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1661
|
+
};
|
|
1662
|
+
return writeRunState(state);
|
|
1663
|
+
},
|
|
1664
|
+
async failRun(runId, error) {
|
|
1665
|
+
const previous = await getRun(runId);
|
|
1666
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1667
|
+
if (await exists2(cancellationPath(previous.id))) {
|
|
1668
|
+
return preserveCancellationRequestState(previous.id);
|
|
1669
|
+
}
|
|
1670
|
+
const state = {
|
|
1671
|
+
...previous,
|
|
1672
|
+
status: "failed",
|
|
1673
|
+
current_stage: "failed",
|
|
1674
|
+
status_message: error,
|
|
1675
|
+
error,
|
|
1676
|
+
completed_at: now,
|
|
1677
|
+
updated_at: now
|
|
1678
|
+
};
|
|
1679
|
+
await writeRunState(state);
|
|
1680
|
+
if (await exists2(cancellationPath(previous.id))) {
|
|
1681
|
+
return preserveCancellationRequestState(previous.id);
|
|
1682
|
+
}
|
|
1683
|
+
await appendRunEvent(state, { stage: "failed", status: "failed", message: error });
|
|
1684
|
+
return state;
|
|
1685
|
+
},
|
|
1686
|
+
async cancelRun(runId) {
|
|
1687
|
+
const state = await getRun(runId);
|
|
1688
|
+
if (isTerminalRunState(state) || state.status === "cancelled" && state.current_stage === "cancel_requested") {
|
|
1689
|
+
return;
|
|
1690
|
+
}
|
|
1691
|
+
await writeFile3(cancellationPath(state.id), `${(/* @__PURE__ */ new Date()).toISOString()}
|
|
1692
|
+
`, "utf8");
|
|
1693
|
+
const latest = await getRun(state.id);
|
|
1694
|
+
if (isTerminalRunState(latest)) {
|
|
1695
|
+
if (latest.status !== "cancelled") await rm(cancellationPath(state.id), { force: true });
|
|
1696
|
+
return;
|
|
1697
|
+
}
|
|
1698
|
+
const updated = {
|
|
1699
|
+
...latest,
|
|
1700
|
+
status: "cancelled",
|
|
1701
|
+
current_stage: "cancel_requested",
|
|
1702
|
+
status_message: "Cancellation requested; waiting for the active worker to stop.",
|
|
1703
|
+
error: void 0,
|
|
1704
|
+
completed_at: void 0,
|
|
1705
|
+
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1706
|
+
};
|
|
1707
|
+
await writeRunState(updated);
|
|
1708
|
+
await appendRunEvent(updated, { stage: "cancel_requested", status: "cancelled", message: "Cancellation requested." });
|
|
1709
|
+
},
|
|
1710
|
+
async finalizeCancellation(runId) {
|
|
1711
|
+
return finalizeCancellationState(runId);
|
|
1712
|
+
},
|
|
1713
|
+
async isCancellationRequested(runId) {
|
|
1714
|
+
const state = await getRun(runId);
|
|
1715
|
+
return state.status === "cancelled" || await exists2(cancellationPath(state.id));
|
|
1716
|
+
},
|
|
1717
|
+
async listRuns() {
|
|
1718
|
+
return listRunRecords();
|
|
1719
|
+
},
|
|
1720
|
+
getRun,
|
|
1721
|
+
async getRunEvents(id) {
|
|
1722
|
+
const state = await getRun(id);
|
|
1723
|
+
return readJsonl(runEventsPath(state.id));
|
|
1724
|
+
},
|
|
1725
|
+
async getRunReport(id) {
|
|
1726
|
+
const state = await getRun(id);
|
|
1727
|
+
if (state.report_path && await exists2(state.report_path)) {
|
|
1728
|
+
return readJson2(state.report_path);
|
|
1729
|
+
}
|
|
1730
|
+
const copiedReportPath = runReportPath(state.id);
|
|
1731
|
+
if (await exists2(copiedReportPath)) return readJson2(copiedReportPath);
|
|
1732
|
+
throw new Error(`Run has no report yet: ${id}`);
|
|
1733
|
+
},
|
|
1734
|
+
async listModels() {
|
|
1735
|
+
return listModelRecords();
|
|
1736
|
+
},
|
|
1737
|
+
async getModel(id) {
|
|
1738
|
+
const record = findByIdOrPrefix(await listModelRecords(), id);
|
|
1739
|
+
if (!record) throw new Error(`Model not found: ${id}`);
|
|
1740
|
+
return record;
|
|
1741
|
+
}
|
|
1742
|
+
};
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
// src/local-runtime/prefetch.ts
|
|
1746
|
+
import { lstat as lstat3, mkdir as mkdir5, readFile as readFile5, readdir as readdir4, stat as stat4 } from "fs/promises";
|
|
1747
|
+
import { join as join5, resolve as resolve8 } from "path";
|
|
1748
|
+
|
|
1749
|
+
// src/local-runtime/process-runner.ts
|
|
1750
|
+
import { createHash as createHash3 } from "crypto";
|
|
1751
|
+
import { spawn } from "child_process";
|
|
1752
|
+
import { createWriteStream, readFileSync } from "fs";
|
|
1753
|
+
import { mkdir as mkdir4 } from "fs/promises";
|
|
1754
|
+
import { homedir as homedir3 } from "os";
|
|
1755
|
+
import { dirname as dirname5, join as join3, resolve as resolve6 } from "path";
|
|
1756
|
+
var packageRoot = localRuntimePackageRoot(import.meta.url);
|
|
1757
|
+
var bundledProject = join3(packageRoot, "training/local-runner");
|
|
1758
|
+
var bundledRuntimeHash = (() => {
|
|
1759
|
+
const hash = createHash3("sha256");
|
|
1760
|
+
for (const name of ["pyproject.toml", "uv.lock"]) {
|
|
1761
|
+
hash.update(readFileSync(join3(bundledProject, name)));
|
|
1762
|
+
}
|
|
1763
|
+
return hash.digest("hex").slice(0, 20);
|
|
1764
|
+
})();
|
|
1765
|
+
var BUNDLED_PYTHON_ENVIRONMENT = join3(
|
|
1766
|
+
resolve6(process.env.XDG_CACHE_HOME ?? join3(homedir3(), ".cache")),
|
|
1767
|
+
"tuned-tensor-local",
|
|
1768
|
+
"uv",
|
|
1769
|
+
bundledRuntimeHash
|
|
1770
|
+
);
|
|
1771
|
+
function buildBundledPythonCommand(entrypoint, args = []) {
|
|
1772
|
+
const target = entrypoint === "-c" ? entrypoint : join3(bundledProject, "src", entrypoint);
|
|
1773
|
+
const commandArgs = [
|
|
1774
|
+
"run",
|
|
1775
|
+
"--frozen",
|
|
1776
|
+
"--project",
|
|
1777
|
+
bundledProject,
|
|
1778
|
+
"python",
|
|
1779
|
+
target,
|
|
1780
|
+
...args
|
|
1781
|
+
];
|
|
1782
|
+
return {
|
|
1783
|
+
command: "uv",
|
|
1784
|
+
commandArgs,
|
|
1785
|
+
displayCommand: ["uv", ...commandArgs]
|
|
1786
|
+
};
|
|
1787
|
+
}
|
|
1788
|
+
function withBundledPythonEnvironment(env) {
|
|
1789
|
+
return {
|
|
1790
|
+
...env,
|
|
1791
|
+
UV_PROJECT_ENVIRONMENT: BUNDLED_PYTHON_ENVIRONMENT
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
var ProcessCancelledError = class extends Error {
|
|
1795
|
+
constructor(message = "Process cancelled.") {
|
|
1796
|
+
super(message);
|
|
1797
|
+
this.name = "ProcessCancelledError";
|
|
1798
|
+
}
|
|
1799
|
+
};
|
|
1800
|
+
async function openProcessLog(path, exclusive) {
|
|
1801
|
+
const stream = createWriteStream(path, {
|
|
1802
|
+
flags: exclusive ? "wx" : "w",
|
|
1803
|
+
mode: 384
|
|
1804
|
+
});
|
|
1805
|
+
await new Promise((resolveOpen, reject) => {
|
|
1806
|
+
const onOpen = () => {
|
|
1807
|
+
stream.off("error", onError);
|
|
1808
|
+
resolveOpen();
|
|
1809
|
+
};
|
|
1810
|
+
const onError = (error) => {
|
|
1811
|
+
stream.off("open", onOpen);
|
|
1812
|
+
reject(error);
|
|
1813
|
+
};
|
|
1814
|
+
stream.once("open", onOpen);
|
|
1815
|
+
stream.once("error", onError);
|
|
1816
|
+
});
|
|
1817
|
+
return stream;
|
|
1818
|
+
}
|
|
1819
|
+
async function runLoggedProcess(args) {
|
|
1820
|
+
if (args.logPath) await mkdir4(dirname5(args.logPath), { recursive: true });
|
|
1821
|
+
const logStream = args.logPath ? await openProcessLog(args.logPath, args.exclusiveLog ?? false) : null;
|
|
1822
|
+
let stderr = "";
|
|
1823
|
+
try {
|
|
1824
|
+
const exitCode = await new Promise((resolvePromise, reject) => {
|
|
1825
|
+
const child = spawn(args.command, args.commandArgs, {
|
|
1826
|
+
cwd: args.cwd ? resolve6(args.cwd) : process.cwd(),
|
|
1827
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1828
|
+
env: args.env ?? process.env,
|
|
1829
|
+
detached: process.platform !== "win32"
|
|
1830
|
+
});
|
|
1831
|
+
let timedOut = false;
|
|
1832
|
+
let cancelled = false;
|
|
1833
|
+
let cancellationError;
|
|
1834
|
+
let cancellationCheckRunning = false;
|
|
1835
|
+
let forceKillTimer = null;
|
|
1836
|
+
const killProcessGroup = (signal) => {
|
|
1837
|
+
if (child.pid && process.platform !== "win32") {
|
|
1838
|
+
try {
|
|
1839
|
+
process.kill(-child.pid, signal);
|
|
1840
|
+
return;
|
|
1841
|
+
} catch {
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
child.kill(signal);
|
|
1845
|
+
};
|
|
1846
|
+
const requestStop = (signal = "SIGTERM") => {
|
|
1847
|
+
killProcessGroup(signal);
|
|
1848
|
+
if (!forceKillTimer) {
|
|
1849
|
+
forceKillTimer = setTimeout(() => killProcessGroup("SIGKILL"), 5e3);
|
|
1850
|
+
forceKillTimer.unref();
|
|
1851
|
+
}
|
|
1852
|
+
};
|
|
1853
|
+
const onSigint = () => {
|
|
1854
|
+
cancelled = true;
|
|
1855
|
+
requestStop("SIGINT");
|
|
1856
|
+
};
|
|
1857
|
+
const onSigterm = () => {
|
|
1858
|
+
cancelled = true;
|
|
1859
|
+
requestStop("SIGTERM");
|
|
1860
|
+
};
|
|
1861
|
+
process.once("SIGINT", onSigint);
|
|
1862
|
+
process.once("SIGTERM", onSigterm);
|
|
1863
|
+
const timer = args.timeoutMs ? setTimeout(() => {
|
|
1864
|
+
timedOut = true;
|
|
1865
|
+
requestStop();
|
|
1866
|
+
}, args.timeoutMs) : null;
|
|
1867
|
+
const cancellationTimer = args.shouldCancel ? setInterval(() => {
|
|
1868
|
+
if (cancellationCheckRunning || cancelled) return;
|
|
1869
|
+
cancellationCheckRunning = true;
|
|
1870
|
+
Promise.resolve(args.shouldCancel?.()).then((requested) => {
|
|
1871
|
+
if (!requested || cancelled) return;
|
|
1872
|
+
cancelled = true;
|
|
1873
|
+
requestStop();
|
|
1874
|
+
}).catch((error) => {
|
|
1875
|
+
cancellationError = error;
|
|
1876
|
+
requestStop();
|
|
1877
|
+
}).finally(() => {
|
|
1878
|
+
cancellationCheckRunning = false;
|
|
1879
|
+
});
|
|
1880
|
+
}, args.cancelPollMs ?? 250) : null;
|
|
1881
|
+
cancellationTimer?.unref();
|
|
1882
|
+
const clearProcessTimers = () => {
|
|
1883
|
+
if (timer) clearTimeout(timer);
|
|
1884
|
+
if (cancellationTimer) clearInterval(cancellationTimer);
|
|
1885
|
+
if (forceKillTimer) clearTimeout(forceKillTimer);
|
|
1886
|
+
process.off("SIGINT", onSigint);
|
|
1887
|
+
process.off("SIGTERM", onSigterm);
|
|
1888
|
+
};
|
|
1889
|
+
if (logStream) {
|
|
1890
|
+
child.stdout.pipe(logStream, { end: false });
|
|
1891
|
+
child.stderr.pipe(logStream, { end: false });
|
|
1892
|
+
}
|
|
1893
|
+
forwardStreamLines(child.stdout, (line) => {
|
|
1894
|
+
args.onLine?.(line, "stdout");
|
|
1895
|
+
if (args.reporter?.verbose) {
|
|
1896
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stdout", message: line }));
|
|
1897
|
+
}
|
|
1898
|
+
});
|
|
1899
|
+
forwardStreamLines(child.stderr, (line) => {
|
|
1900
|
+
stderr += `${line}
|
|
1901
|
+
`;
|
|
1902
|
+
args.onLine?.(line, "stderr");
|
|
1903
|
+
if (args.reporter?.verbose) {
|
|
1904
|
+
reportInBackground(() => args.reporter?.onLog?.({ stage: args.stage, stream: "stderr", message: line }));
|
|
1905
|
+
}
|
|
1906
|
+
});
|
|
1907
|
+
child.on("error", (error) => {
|
|
1908
|
+
clearProcessTimers();
|
|
1909
|
+
reject(timedOut ? new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`) : error);
|
|
1910
|
+
});
|
|
1911
|
+
child.on("close", (code) => {
|
|
1912
|
+
if (timedOut || cancelled || cancellationError || args.terminateProcessGroupOnExit) {
|
|
1913
|
+
killProcessGroup("SIGKILL");
|
|
1914
|
+
}
|
|
1915
|
+
clearProcessTimers();
|
|
1916
|
+
if (timedOut) {
|
|
1917
|
+
reject(new Error(args.timeoutMessage ?? `${args.command} timed out after ${args.timeoutMs}ms`));
|
|
1918
|
+
return;
|
|
1919
|
+
}
|
|
1920
|
+
if (cancellationError) {
|
|
1921
|
+
reject(cancellationError);
|
|
1922
|
+
return;
|
|
1923
|
+
}
|
|
1924
|
+
if (cancelled) {
|
|
1925
|
+
reject(new ProcessCancelledError(`${args.stage} was cancelled.`));
|
|
1926
|
+
return;
|
|
1927
|
+
}
|
|
1928
|
+
resolvePromise(code ?? 1);
|
|
1929
|
+
});
|
|
1930
|
+
});
|
|
1931
|
+
return { exitCode, stderr };
|
|
1932
|
+
} finally {
|
|
1933
|
+
if (logStream) {
|
|
1934
|
+
await new Promise((resolveEnd) => {
|
|
1935
|
+
logStream.end(resolveEnd);
|
|
1936
|
+
});
|
|
1937
|
+
}
|
|
1938
|
+
}
|
|
1939
|
+
}
|
|
1940
|
+
|
|
1941
|
+
// src/local-runtime/huggingface-cache.ts
|
|
1942
|
+
import { homedir as homedir4 } from "os";
|
|
1943
|
+
import { dirname as dirname6, join as join4, resolve as resolve7 } from "path";
|
|
1944
|
+
function minimalMachineLearningEnvironment(env) {
|
|
1945
|
+
const exact = /* @__PURE__ */ new Set([
|
|
1946
|
+
"PATH",
|
|
1947
|
+
"HOME",
|
|
1948
|
+
"USER",
|
|
1949
|
+
"LOGNAME",
|
|
1950
|
+
"SHELL",
|
|
1951
|
+
"TMPDIR",
|
|
1952
|
+
"TEMP",
|
|
1953
|
+
"TMP",
|
|
1954
|
+
"LANG",
|
|
1955
|
+
"LANGUAGE",
|
|
1956
|
+
"LD_LIBRARY_PATH",
|
|
1957
|
+
"DYLD_LIBRARY_PATH",
|
|
1958
|
+
"XDG_CACHE_HOME",
|
|
1959
|
+
"SSL_CERT_FILE",
|
|
1960
|
+
"REQUESTS_CA_BUNDLE",
|
|
1961
|
+
"CURL_CA_BUNDLE",
|
|
1962
|
+
"NO_PROXY",
|
|
1963
|
+
"no_proxy",
|
|
1964
|
+
"UV_CACHE_DIR",
|
|
1965
|
+
"UV_PYTHON",
|
|
1966
|
+
"HF_HOME",
|
|
1967
|
+
"HF_HUB_CACHE",
|
|
1968
|
+
"HUGGINGFACE_HUB_CACHE",
|
|
1969
|
+
"HF_ENDPOINT",
|
|
1970
|
+
"HF_TOKEN",
|
|
1971
|
+
"HF_HUB_OFFLINE",
|
|
1972
|
+
"HF_HUB_DISABLE_TELEMETRY",
|
|
1973
|
+
"TRANSFORMERS_OFFLINE",
|
|
1974
|
+
"HF_HUB_ENABLE_HF_TRANSFER",
|
|
1975
|
+
"PYTHONUTF8",
|
|
1976
|
+
"PYTHONIOENCODING"
|
|
1977
|
+
]);
|
|
1978
|
+
const prefixes = [
|
|
1979
|
+
"LC_",
|
|
1980
|
+
"CUDA_",
|
|
1981
|
+
"NVIDIA_",
|
|
1982
|
+
"NCCL_",
|
|
1983
|
+
"OMP_",
|
|
1984
|
+
"MKL_",
|
|
1985
|
+
"TORCH_",
|
|
1986
|
+
"PYTORCH_",
|
|
1987
|
+
"TOKENIZERS_",
|
|
1988
|
+
"HF_XET_"
|
|
1989
|
+
];
|
|
1990
|
+
const result = Object.fromEntries(Object.entries(env).filter(
|
|
1991
|
+
([key]) => exact.has(key) || prefixes.some((prefix) => key.startsWith(prefix))
|
|
1992
|
+
));
|
|
1993
|
+
result.TORCH_DISABLE_NATIVE_JIT ??= "1";
|
|
1994
|
+
return result;
|
|
1995
|
+
}
|
|
1996
|
+
function resolveHuggingFaceCacheLayout(modelCache) {
|
|
1997
|
+
const hfHome = resolve7(modelCache);
|
|
1998
|
+
return {
|
|
1999
|
+
hfHome,
|
|
2000
|
+
hubCache: join4(hfHome, "hub")
|
|
2001
|
+
};
|
|
2002
|
+
}
|
|
2003
|
+
function withHuggingFaceCacheEnvironment(env, modelCache) {
|
|
2004
|
+
const inheritedHub = env.HF_HUB_CACHE ?? env.HUGGINGFACE_HUB_CACHE;
|
|
2005
|
+
const canonicalHome = modelCache ?? env.HF_HOME ?? (inheritedHub ? dirname6(resolve7(inheritedHub)) : join4(homedir4(), ".cache", "huggingface"));
|
|
2006
|
+
const layout = resolveHuggingFaceCacheLayout(canonicalHome);
|
|
2007
|
+
const result = {
|
|
2008
|
+
...env,
|
|
2009
|
+
HF_HOME: layout.hfHome,
|
|
2010
|
+
HF_HUB_CACHE: layout.hubCache,
|
|
2011
|
+
HUGGINGFACE_HUB_CACHE: layout.hubCache
|
|
2012
|
+
};
|
|
2013
|
+
delete result.TRANSFORMERS_CACHE;
|
|
2014
|
+
delete result.PYTORCH_TRANSFORMERS_CACHE;
|
|
2015
|
+
delete result.PYTORCH_PRETRAINED_BERT_CACHE;
|
|
2016
|
+
return result;
|
|
2017
|
+
}
|
|
2018
|
+
function withOfflineHuggingFaceCacheEnvironment(env, modelCache) {
|
|
2019
|
+
return {
|
|
2020
|
+
...withHuggingFaceCacheEnvironment(env, modelCache),
|
|
2021
|
+
HF_HUB_OFFLINE: "1",
|
|
2022
|
+
TRANSFORMERS_OFFLINE: "1"
|
|
2023
|
+
};
|
|
2024
|
+
}
|
|
2025
|
+
|
|
2026
|
+
// src/local-runtime/prefetch.ts
|
|
2027
|
+
function buildModelPrefetchPayload(request, config) {
|
|
2028
|
+
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
2029
|
+
return {
|
|
2030
|
+
base_model: model.id,
|
|
2031
|
+
...request.hyperparameters.base_model_revision ? { revision: request.hyperparameters.base_model_revision } : {},
|
|
2032
|
+
...config.paths.modelCache ? { model_cache: resolve8(config.paths.modelCache) } : {}
|
|
2033
|
+
};
|
|
2034
|
+
}
|
|
2035
|
+
function safeName(value) {
|
|
2036
|
+
return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "model";
|
|
2037
|
+
}
|
|
2038
|
+
function transformersWeightName(name) {
|
|
2039
|
+
const lower = name.split(/[\\/]/).at(-1)?.toLowerCase() ?? "";
|
|
2040
|
+
return lower.endsWith(".safetensors") && !lower.startsWith("adapter_") || /^pytorch_model.*\.bin$/.test(lower);
|
|
2041
|
+
}
|
|
2042
|
+
async function verifyLocalBaseModel(path) {
|
|
2043
|
+
const root = await lstat3(path).catch(() => null);
|
|
2044
|
+
if (!root) throw new Error(`paths.baseModel is set to ${path}, but that path does not exist.`);
|
|
2045
|
+
if (root.isSymbolicLink()) {
|
|
2046
|
+
throw new Error(`paths.baseModel must not itself be a symbolic link: ${path}`);
|
|
2047
|
+
}
|
|
2048
|
+
if (!root.isDirectory()) {
|
|
2049
|
+
throw new Error(
|
|
2050
|
+
`paths.baseModel must be a Hugging Face snapshot directory, not a standalone file or archive: ${path}`
|
|
2051
|
+
);
|
|
2052
|
+
}
|
|
2053
|
+
const files = [];
|
|
2054
|
+
const visit = async (directory) => {
|
|
2055
|
+
for (const entry of await readdir4(directory, { withFileTypes: true })) {
|
|
2056
|
+
const child = join5(directory, entry.name);
|
|
2057
|
+
if (entry.isDirectory()) {
|
|
2058
|
+
await visit(child);
|
|
2059
|
+
continue;
|
|
2060
|
+
}
|
|
2061
|
+
const metadata = await stat4(child);
|
|
2062
|
+
if (entry.isSymbolicLink() && !metadata.isFile()) {
|
|
2063
|
+
throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
|
|
2064
|
+
}
|
|
2065
|
+
if (metadata.isFile()) files.push({ path: child, size: metadata.size });
|
|
2066
|
+
}
|
|
2067
|
+
};
|
|
2068
|
+
await visit(path);
|
|
2069
|
+
const nonEmpty = files.filter((file) => file.size > 0);
|
|
2070
|
+
const weights = nonEmpty.filter((file) => transformersWeightName(file.path));
|
|
2071
|
+
if (weights.length === 0) {
|
|
2072
|
+
throw new Error(`Local base-model directory contains no non-empty Transformers model weights: ${path}`);
|
|
2073
|
+
}
|
|
2074
|
+
const required = async (name) => {
|
|
2075
|
+
const metadata = await stat4(join5(path, name)).catch(() => null);
|
|
2076
|
+
return Boolean(metadata?.isFile() && metadata.size > 0);
|
|
2077
|
+
};
|
|
2078
|
+
if (!await required("config.json")) throw new Error(`Local base-model directory is missing config.json: ${path}`);
|
|
2079
|
+
try {
|
|
2080
|
+
const config = JSON.parse(await readFile5(join5(path, "config.json"), "utf8"));
|
|
2081
|
+
assertCertifiedBaseModelConfig(config, `Local base-model config ${join5(path, "config.json")}`);
|
|
2082
|
+
} catch (error) {
|
|
2083
|
+
throw new Error(`Local base-model directory has an invalid or unsupported config.json: ${path}`, { cause: error });
|
|
2084
|
+
}
|
|
2085
|
+
const vocabNames = [
|
|
2086
|
+
"tokenizer.json",
|
|
2087
|
+
"tokenizer.model",
|
|
2088
|
+
"sentencepiece.bpe.model",
|
|
2089
|
+
"spiece.model",
|
|
2090
|
+
"vocab.json",
|
|
2091
|
+
"tokenizer.tiktoken"
|
|
2092
|
+
];
|
|
2093
|
+
if (!await required("tokenizer_config.json") || !(await Promise.all(vocabNames.map(required))).some(Boolean)) {
|
|
2094
|
+
throw new Error(`Local base-model directory is missing tokenizer metadata or vocabulary: ${path}`);
|
|
2095
|
+
}
|
|
2096
|
+
for (const file of files.filter((candidate) => candidate.path.endsWith(".index.json"))) {
|
|
2097
|
+
const parsed = JSON.parse(await readFile5(file.path, "utf8"));
|
|
2098
|
+
if (!parsed.weight_map || typeof parsed.weight_map !== "object" || Array.isArray(parsed.weight_map)) {
|
|
2099
|
+
throw new Error(`Local base-model weight index is invalid: ${file.path}`);
|
|
2100
|
+
}
|
|
2101
|
+
for (const shard of new Set(Object.values(parsed.weight_map))) {
|
|
2102
|
+
if (typeof shard !== "string" || !await required(shard)) {
|
|
2103
|
+
throw new Error(`Local base-model directory is missing indexed weight shard: ${String(shard)}`);
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
return {
|
|
2108
|
+
fileCount: files.length,
|
|
2109
|
+
sizeBytes: files.reduce((total, file) => total + file.size, 0)
|
|
2110
|
+
};
|
|
2111
|
+
}
|
|
2112
|
+
async function prefetchBaseModel(args) {
|
|
2113
|
+
const artifactDir = resolve8(
|
|
2114
|
+
args.config.artifactRoot,
|
|
2115
|
+
args.localOnly ? "verify-base" : "prefetch",
|
|
2116
|
+
`${safeName(args.request.spec_snapshot.base_model)}-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`
|
|
2117
|
+
);
|
|
2118
|
+
await mkdir5(artifactDir, { recursive: true });
|
|
2119
|
+
if (args.config.paths.baseModel) {
|
|
2120
|
+
const localPath = resolve8(args.config.paths.baseModel);
|
|
2121
|
+
const verified = await verifyLocalBaseModel(localPath);
|
|
2122
|
+
return {
|
|
2123
|
+
ok: true,
|
|
2124
|
+
status: args.localOnly ? "completed" : "skipped",
|
|
2125
|
+
base_model: args.request.spec_snapshot.base_model,
|
|
2126
|
+
local_base_model_path: localPath,
|
|
2127
|
+
file_count: verified.fileCount,
|
|
2128
|
+
size_bytes: verified.sizeBytes,
|
|
2129
|
+
artifact_dir: artifactDir,
|
|
2130
|
+
reason: args.localOnly ? "Verified the configured local base-model artifact; no network access was used." : "paths.baseModel points at a verified local base-model artifact; no Hugging Face download is needed."
|
|
2131
|
+
};
|
|
2132
|
+
}
|
|
2133
|
+
const payload = {
|
|
2134
|
+
...buildModelPrefetchPayload(args.request, args.config),
|
|
2135
|
+
...args.localOnly ? { local_files_only: true } : {}
|
|
2136
|
+
};
|
|
2137
|
+
const inputPath = join5(artifactDir, "prefetch-input.json");
|
|
2138
|
+
const outputPath = join5(artifactDir, "prefetch-output.json");
|
|
2139
|
+
const logPath = join5(artifactDir, "prefetch.log");
|
|
2140
|
+
await writeJson(inputPath, payload);
|
|
2141
|
+
const entrypoint = buildBundledPythonCommand(
|
|
2142
|
+
"prefetch.py",
|
|
2143
|
+
["--input", inputPath, "--output", outputPath]
|
|
2144
|
+
);
|
|
2145
|
+
await args.reporter?.onEvent?.({
|
|
2146
|
+
stage: "model_prefetch",
|
|
2147
|
+
status: "running",
|
|
2148
|
+
message: args.localOnly ? "Verifying the Hugging Face base model is available in the local cache." : "Prefetching Hugging Face base model.",
|
|
2149
|
+
details: {
|
|
2150
|
+
base_model: payload.base_model,
|
|
2151
|
+
model_cache: payload.model_cache ?? null,
|
|
2152
|
+
command: entrypoint.displayCommand,
|
|
2153
|
+
log_path: logPath
|
|
2154
|
+
}
|
|
2155
|
+
});
|
|
2156
|
+
const env = withBundledPythonEnvironment(
|
|
2157
|
+
withHuggingFaceCacheEnvironment(
|
|
2158
|
+
minimalMachineLearningEnvironment(process.env),
|
|
2159
|
+
payload.model_cache
|
|
2160
|
+
)
|
|
2161
|
+
);
|
|
2162
|
+
const { exitCode } = await runLoggedProcess({
|
|
2163
|
+
command: entrypoint.command,
|
|
2164
|
+
commandArgs: entrypoint.commandArgs,
|
|
2165
|
+
env,
|
|
2166
|
+
logPath,
|
|
2167
|
+
// A base model can be several gigabytes. Surface the downloader's progress
|
|
2168
|
+
// by default; callers can still opt out by omitting the reporter (`--quiet`).
|
|
2169
|
+
reporter: args.reporter ? { ...args.reporter, verbose: true } : void 0,
|
|
2170
|
+
stage: "model_prefetch"
|
|
2171
|
+
});
|
|
2172
|
+
if (exitCode !== 0) {
|
|
2173
|
+
throw new Error(`Model prefetch exited with code ${exitCode}. See ${logPath}.`);
|
|
2174
|
+
}
|
|
2175
|
+
const output = JSON.parse(await readFile5(outputPath, "utf8"));
|
|
2176
|
+
const snapshotRevision = baseModelRevisionSchema.safeParse(output.snapshot_revision);
|
|
2177
|
+
if (!snapshotRevision.success) {
|
|
2178
|
+
throw new Error(
|
|
2179
|
+
"Model prefetch did not resolve the base model to a 40-character immutable commit SHA."
|
|
2180
|
+
);
|
|
2181
|
+
}
|
|
2182
|
+
await args.reporter?.onEvent?.({
|
|
2183
|
+
stage: "model_prefetch",
|
|
2184
|
+
status: "completed",
|
|
2185
|
+
message: "Base model is available in the local Hugging Face cache.",
|
|
2186
|
+
details: {
|
|
2187
|
+
base_model: output.base_model ?? payload.base_model,
|
|
2188
|
+
hf_home: output.hf_home ?? output.model_cache ?? payload.model_cache ?? null,
|
|
2189
|
+
hub_cache: output.hub_cache ?? null,
|
|
2190
|
+
snapshot_path: output.snapshot_path ?? null,
|
|
2191
|
+
snapshot_revision: snapshotRevision.data,
|
|
2192
|
+
file_count: output.file_count ?? null,
|
|
2193
|
+
size_bytes: output.size_bytes ?? null,
|
|
2194
|
+
verified_blob_count: output.verified_blob_count ?? null
|
|
2195
|
+
}
|
|
2196
|
+
});
|
|
2197
|
+
return {
|
|
2198
|
+
ok: true,
|
|
2199
|
+
status: "completed",
|
|
2200
|
+
base_model: output.base_model ?? payload.base_model,
|
|
2201
|
+
model_cache: output.model_cache ?? payload.model_cache,
|
|
2202
|
+
hf_home: output.hf_home ?? output.model_cache ?? payload.model_cache,
|
|
2203
|
+
hub_cache: output.hub_cache,
|
|
2204
|
+
snapshot_path: output.snapshot_path,
|
|
2205
|
+
snapshot_revision: snapshotRevision.data,
|
|
2206
|
+
file_count: output.file_count,
|
|
2207
|
+
size_bytes: output.size_bytes,
|
|
2208
|
+
verified_blob_count: output.verified_blob_count,
|
|
2209
|
+
artifact_dir: artifactDir,
|
|
2210
|
+
input_path: inputPath,
|
|
2211
|
+
output_path: outputPath,
|
|
2212
|
+
log_uri: fileUri(logPath),
|
|
2213
|
+
command: entrypoint.displayCommand
|
|
2214
|
+
};
|
|
2215
|
+
}
|
|
2216
|
+
|
|
2217
|
+
// src/local-runtime/general-regression.ts
|
|
2218
|
+
function evaluateGeneralRegressionGate(comparison, policy) {
|
|
2219
|
+
const failures = [];
|
|
2220
|
+
const scoreDrop = Math.max(0, -comparison.avg_score_delta);
|
|
2221
|
+
const passRateDrop = Math.max(0, -comparison.pass_rate_delta);
|
|
2222
|
+
const tolerance = 1e-12;
|
|
2223
|
+
if (scoreDrop > policy.maxScoreDrop + tolerance) {
|
|
2224
|
+
failures.push(
|
|
2225
|
+
`Average score dropped ${scoreDrop.toFixed(4)}, exceeding the ${policy.maxScoreDrop.toFixed(4)} budget.`
|
|
2226
|
+
);
|
|
2227
|
+
}
|
|
2228
|
+
if (passRateDrop > policy.maxPassRateDrop + tolerance) {
|
|
2229
|
+
failures.push(
|
|
2230
|
+
`Pass rate dropped ${passRateDrop.toFixed(4)}, exceeding the ${policy.maxPassRateDrop.toFixed(4)} budget.`
|
|
2231
|
+
);
|
|
2232
|
+
}
|
|
2233
|
+
return { passed: failures.length === 0, failures };
|
|
2234
|
+
}
|
|
2235
|
+
|
|
2236
|
+
// src/local-runtime/orchestrator.ts
|
|
2237
|
+
import { lstat as lstat4, mkdir as mkdir8, open as open2, readFile as readFile8, readdir as readdir5, rename as rename3, rm as rm3, stat as stat5 } from "fs/promises";
|
|
2238
|
+
import { createReadStream as createReadStream2 } from "fs";
|
|
2239
|
+
import { createHash as createHash5, randomUUID as randomUUID4 } from "crypto";
|
|
2240
|
+
import { performance } from "perf_hooks";
|
|
2241
|
+
import { homedir as homedir5 } from "os";
|
|
2242
|
+
import { basename, dirname as dirname8, join as join8, relative as relative2, resolve as resolve11 } from "path";
|
|
2243
|
+
|
|
2244
|
+
// src/local-runtime/evaluation.ts
|
|
2245
|
+
import { createHash as createHash4 } from "crypto";
|
|
2246
|
+
import { mkdir as mkdir6, readFile as readFile6, rm as rm2, writeFile as writeFile4 } from "fs/promises";
|
|
2247
|
+
import { dirname as dirname7, join as join6, resolve as resolve9 } from "path";
|
|
2248
|
+
function normalize(value) {
|
|
2249
|
+
return value.trim().replace(/\s+/g, " ").toLowerCase();
|
|
2250
|
+
}
|
|
2251
|
+
function exactScore(expected, actual) {
|
|
2252
|
+
return normalize(expected) === normalize(actual) ? 1 : 0;
|
|
2253
|
+
}
|
|
2254
|
+
function tokenF1(expected, actual) {
|
|
2255
|
+
const tokenize = (value) => normalize(value).match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
2256
|
+
const expectedTokens = tokenize(expected);
|
|
2257
|
+
const actualTokens = tokenize(actual);
|
|
2258
|
+
if (expectedTokens.length === 0 || actualTokens.length === 0) {
|
|
2259
|
+
return expectedTokens.length === actualTokens.length ? 1 : 0;
|
|
2260
|
+
}
|
|
2261
|
+
const counts = /* @__PURE__ */ new Map();
|
|
2262
|
+
for (const token of expectedTokens) {
|
|
2263
|
+
counts.set(token, (counts.get(token) ?? 0) + 1);
|
|
2264
|
+
}
|
|
2265
|
+
let overlap = 0;
|
|
2266
|
+
for (const token of actualTokens) {
|
|
2267
|
+
const remaining = counts.get(token) ?? 0;
|
|
2268
|
+
if (remaining > 0) {
|
|
2269
|
+
overlap += 1;
|
|
2270
|
+
counts.set(token, remaining - 1);
|
|
2271
|
+
}
|
|
2272
|
+
}
|
|
2273
|
+
if (overlap === 0) return 0;
|
|
2274
|
+
const precision = overlap / actualTokens.length;
|
|
2275
|
+
const recall = overlap / expectedTokens.length;
|
|
2276
|
+
return 2 * precision * recall / (precision + recall);
|
|
2277
|
+
}
|
|
2278
|
+
function deriveSampleSeed(value) {
|
|
2279
|
+
let hash = 2166136261;
|
|
2280
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
2281
|
+
hash ^= value.charCodeAt(index);
|
|
2282
|
+
hash = Math.imul(hash, 16777619);
|
|
2283
|
+
}
|
|
2284
|
+
return hash >>> 0;
|
|
2285
|
+
}
|
|
2286
|
+
function mulberry32(seed) {
|
|
2287
|
+
let state = seed >>> 0;
|
|
2288
|
+
return () => {
|
|
2289
|
+
state = state + 1831565813 | 0;
|
|
2290
|
+
let value = Math.imul(state ^ state >>> 15, 1 | state);
|
|
2291
|
+
value = value + Math.imul(value ^ value >>> 7, 61 | value) ^ value;
|
|
2292
|
+
return ((value ^ value >>> 14) >>> 0) / 4294967296;
|
|
2293
|
+
};
|
|
2294
|
+
}
|
|
2295
|
+
function sampleExamples(examples, count, seed) {
|
|
2296
|
+
if (count >= examples.length) return examples;
|
|
2297
|
+
const indices = examples.map((_, index) => index);
|
|
2298
|
+
const random = mulberry32(seed);
|
|
2299
|
+
for (let index = indices.length - 1; index > 0; index -= 1) {
|
|
2300
|
+
const swap = Math.floor(random() * (index + 1));
|
|
2301
|
+
[indices[index], indices[swap]] = [indices[swap], indices[index]];
|
|
2302
|
+
}
|
|
2303
|
+
return indices.slice(0, count).sort((left, right) => left - right).map((index) => examples[index]);
|
|
2304
|
+
}
|
|
2305
|
+
function splitSpecExamples(examples, seed, holdoutRatio = 0.2) {
|
|
2306
|
+
if (examples.length < 2) return { train: examples, holdout: [] };
|
|
2307
|
+
const holdoutCount = Math.min(
|
|
2308
|
+
examples.length - 1,
|
|
2309
|
+
Math.max(1, Math.round(examples.length * holdoutRatio))
|
|
2310
|
+
);
|
|
2311
|
+
const indices = examples.map((_, index) => index);
|
|
2312
|
+
const random = mulberry32(seed);
|
|
2313
|
+
for (let index = indices.length - 1; index > 0; index -= 1) {
|
|
2314
|
+
const swap = Math.floor(random() * (index + 1));
|
|
2315
|
+
[indices[index], indices[swap]] = [indices[swap], indices[index]];
|
|
2316
|
+
}
|
|
2317
|
+
const heldOut = new Set(indices.slice(0, holdoutCount));
|
|
2318
|
+
return {
|
|
2319
|
+
train: examples.filter((_, index) => !heldOut.has(index)),
|
|
2320
|
+
holdout: examples.filter((_, index) => heldOut.has(index))
|
|
2321
|
+
};
|
|
2322
|
+
}
|
|
2323
|
+
function extractJsonObject(value) {
|
|
2324
|
+
const trimmed = value.trim();
|
|
2325
|
+
const candidates = [trimmed];
|
|
2326
|
+
const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
|
|
2327
|
+
if (fenced?.[1]) candidates.push(fenced[1].trim());
|
|
2328
|
+
const objectLike = trimmed.match(/\{[\s\S]*\}/);
|
|
2329
|
+
if (objectLike?.[0]) candidates.push(objectLike[0]);
|
|
2330
|
+
for (const candidate of candidates) {
|
|
2331
|
+
try {
|
|
2332
|
+
const parsed = JSON.parse(candidate);
|
|
2333
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
2334
|
+
return parsed;
|
|
2335
|
+
}
|
|
2336
|
+
} catch {
|
|
2337
|
+
}
|
|
2338
|
+
}
|
|
2339
|
+
return null;
|
|
2340
|
+
}
|
|
2341
|
+
function canonicalJson(value) {
|
|
2342
|
+
if (Array.isArray(value)) return value.map(canonicalJson);
|
|
2343
|
+
if (value && typeof value === "object") {
|
|
2344
|
+
return Object.fromEntries(
|
|
2345
|
+
Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, item]) => [key, canonicalJson(item)])
|
|
2346
|
+
);
|
|
2347
|
+
}
|
|
2348
|
+
return value;
|
|
2349
|
+
}
|
|
2350
|
+
function scoreJsonFields(expected, actual, configuredFields) {
|
|
2351
|
+
const expectedJson = extractJsonObject(expected);
|
|
2352
|
+
const actualJson = extractJsonObject(actual);
|
|
2353
|
+
if (!expectedJson) {
|
|
2354
|
+
const score2 = exactScore(expected, actual);
|
|
2355
|
+
return {
|
|
2356
|
+
score: score2,
|
|
2357
|
+
passed: score2 === 1,
|
|
2358
|
+
reasoning: "Expected output is not JSON; used normalized exact match.",
|
|
2359
|
+
actualJsonValid: Boolean(actualJson),
|
|
2360
|
+
schemaMatch: false,
|
|
2361
|
+
fields: [],
|
|
2362
|
+
fieldResults: {}
|
|
2363
|
+
};
|
|
2364
|
+
}
|
|
2365
|
+
const fields = configuredFields?.length ? configuredFields : Object.keys(expectedJson).sort();
|
|
2366
|
+
const expectedKeys = Object.keys(expectedJson).sort();
|
|
2367
|
+
const actualKeys = actualJson ? Object.keys(actualJson).sort() : [];
|
|
2368
|
+
const schemaMatch = Boolean(actualJson) && expectedKeys.length === actualKeys.length && expectedKeys.every((key, index) => key === actualKeys[index]);
|
|
2369
|
+
const fieldResults = {};
|
|
2370
|
+
for (const field of fields) {
|
|
2371
|
+
fieldResults[field] = Object.prototype.hasOwnProperty.call(expectedJson, field) && Boolean(actualJson) && JSON.stringify(canonicalJson(expectedJson[field])) === JSON.stringify(canonicalJson(actualJson?.[field]));
|
|
2372
|
+
}
|
|
2373
|
+
const correct = Object.values(fieldResults).filter(Boolean).length;
|
|
2374
|
+
const score = fields.length > 0 ? correct / fields.length : 0;
|
|
2375
|
+
return {
|
|
2376
|
+
score,
|
|
2377
|
+
passed: fields.length > 0 && correct === fields.length,
|
|
2378
|
+
reasoning: actualJson ? `JSON field score: ${correct}/${fields.length} fields matched.` : "Actual output is not a JSON object.",
|
|
2379
|
+
actualJsonValid: Boolean(actualJson),
|
|
2380
|
+
schemaMatch,
|
|
2381
|
+
fields,
|
|
2382
|
+
fieldResults
|
|
2383
|
+
};
|
|
2384
|
+
}
|
|
2385
|
+
function aggregateJsonFieldMetrics(scores, total) {
|
|
2386
|
+
if (total === 0 || scores.length === 0) return void 0;
|
|
2387
|
+
const fields = [...new Set(scores.flatMap((score) => score.fields))].sort();
|
|
2388
|
+
const field_accuracy = {};
|
|
2389
|
+
for (const field of fields) {
|
|
2390
|
+
const applicable = scores.filter((score) => score.fields.includes(field));
|
|
2391
|
+
const correct = applicable.filter((score) => score.fieldResults[field]).length;
|
|
2392
|
+
field_accuracy[field] = {
|
|
2393
|
+
correct,
|
|
2394
|
+
total: applicable.length,
|
|
2395
|
+
accuracy: applicable.length > 0 ? correct / applicable.length : 0
|
|
2396
|
+
};
|
|
2397
|
+
}
|
|
2398
|
+
const validJsonCount = scores.filter((score) => score.actualJsonValid).length;
|
|
2399
|
+
const schemaMatchCount = scores.filter((score) => score.schemaMatch).length;
|
|
2400
|
+
const allFieldsMatchCount = scores.filter((score) => score.passed).length;
|
|
2401
|
+
return {
|
|
2402
|
+
fields,
|
|
2403
|
+
valid_json_count: validJsonCount,
|
|
2404
|
+
valid_json_rate: validJsonCount / total,
|
|
2405
|
+
schema_match_count: schemaMatchCount,
|
|
2406
|
+
schema_match_rate: schemaMatchCount / total,
|
|
2407
|
+
all_fields_match_count: allFieldsMatchCount,
|
|
2408
|
+
all_fields_match_rate: allFieldsMatchCount / total,
|
|
2409
|
+
field_accuracy
|
|
2410
|
+
};
|
|
2411
|
+
}
|
|
2412
|
+
function fileUriToPath(value) {
|
|
2413
|
+
if (!value) return void 0;
|
|
2414
|
+
return value.startsWith("file://") ? value.slice("file://".length) : value;
|
|
2415
|
+
}
|
|
2416
|
+
var INFERENCE_PROTOCOL_VERSION = 2;
|
|
2417
|
+
function parseBatchInferenceResult(value, expectedIds) {
|
|
2418
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
2419
|
+
throw new Error("Transformers inference output must be a JSON object");
|
|
2420
|
+
}
|
|
2421
|
+
const output = value;
|
|
2422
|
+
if (!Array.isArray(output.results) || output.results.length !== expectedIds.length) {
|
|
2423
|
+
throw new Error(
|
|
2424
|
+
`Transformers inference returned ${Array.isArray(output.results) ? output.results.length : 0} predictions; expected ${expectedIds.length}`
|
|
2425
|
+
);
|
|
2426
|
+
}
|
|
2427
|
+
const expected = new Set(expectedIds);
|
|
2428
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2429
|
+
for (const [index, entry] of output.results.entries()) {
|
|
2430
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
|
2431
|
+
throw new Error(`Transformers prediction ${index} must be an object`);
|
|
2432
|
+
}
|
|
2433
|
+
const prediction = entry;
|
|
2434
|
+
if (typeof prediction.id !== "string" || !expected.has(prediction.id)) {
|
|
2435
|
+
throw new Error(`Transformers prediction ${index} has an unknown or missing id`);
|
|
2436
|
+
}
|
|
2437
|
+
if (byId.has(prediction.id)) {
|
|
2438
|
+
throw new Error(`Transformers inference returned duplicate id ${prediction.id}`);
|
|
2439
|
+
}
|
|
2440
|
+
if (typeof prediction.actual !== "string") {
|
|
2441
|
+
throw new Error(`Transformers prediction ${index} must include string actual`);
|
|
2442
|
+
}
|
|
2443
|
+
if (typeof prediction.latency_ms !== "number" || !Number.isInteger(prediction.latency_ms) || prediction.latency_ms < 0) {
|
|
2444
|
+
throw new Error(
|
|
2445
|
+
`Transformers prediction ${index} must include non-negative integer latency_ms`
|
|
2446
|
+
);
|
|
2447
|
+
}
|
|
2448
|
+
byId.set(prediction.id, {
|
|
2449
|
+
id: prediction.id,
|
|
2450
|
+
actual: prediction.actual,
|
|
2451
|
+
latency_ms: prediction.latency_ms
|
|
2452
|
+
});
|
|
2453
|
+
}
|
|
2454
|
+
const optionalString = (key) => typeof output[key] === "string" ? output[key] : void 0;
|
|
2455
|
+
return {
|
|
2456
|
+
...optionalString("model_id") ? { model_id: optionalString("model_id") } : {},
|
|
2457
|
+
...optionalString("base_model") ? { base_model: optionalString("base_model") } : {},
|
|
2458
|
+
...optionalString("adapter_path") ? { adapter_path: optionalString("adapter_path") } : {},
|
|
2459
|
+
...output.generation_config && typeof output.generation_config === "object" && !Array.isArray(output.generation_config) ? { generation_config: output.generation_config } : {},
|
|
2460
|
+
results: expectedIds.map((id) => byId.get(id))
|
|
2461
|
+
};
|
|
2462
|
+
}
|
|
2463
|
+
async function runTransformersInference(args) {
|
|
2464
|
+
const inputPath = `${args.outputPath}.inference-input.json`;
|
|
2465
|
+
const outputPath = `${args.outputPath}.inference-output.json`;
|
|
2466
|
+
const logPath = `${args.outputPath}.inference.log`;
|
|
2467
|
+
await mkdir6(dirname7(inputPath), { recursive: true });
|
|
2468
|
+
await writeFile4(inputPath, `${JSON.stringify({
|
|
2469
|
+
protocol_version: INFERENCE_PROTOCOL_VERSION,
|
|
2470
|
+
kind: args.kind,
|
|
2471
|
+
model_id: args.modelId,
|
|
2472
|
+
base_model: args.baseModelId,
|
|
2473
|
+
base_model_revision: args.baseModelRevision,
|
|
2474
|
+
model_loader: "causal_lm",
|
|
2475
|
+
adapter_path: fileUriToPath(args.adapterPath),
|
|
2476
|
+
system: args.system,
|
|
2477
|
+
examples: args.examples.map((example, index) => ({
|
|
2478
|
+
id: String(index),
|
|
2479
|
+
input: example.input
|
|
2480
|
+
})),
|
|
2481
|
+
model_cache: args.config.paths.modelCache ? resolve9(args.config.paths.modelCache) : void 0,
|
|
2482
|
+
trust_remote_code: false,
|
|
2483
|
+
device: args.config.evaluation.inference.device,
|
|
2484
|
+
generation: {
|
|
2485
|
+
max_new_tokens: args.config.evaluation.inference.maxNewTokens,
|
|
2486
|
+
temperature: args.config.evaluation.inference.temperature,
|
|
2487
|
+
top_p: args.config.evaluation.inference.topP
|
|
2488
|
+
}
|
|
2489
|
+
}, null, 2)}
|
|
2490
|
+
`, "utf8");
|
|
2491
|
+
await rm2(outputPath, { force: true });
|
|
2492
|
+
const entrypoint = buildBundledPythonCommand(
|
|
2493
|
+
"evaluate.py",
|
|
2494
|
+
["--input", inputPath, "--output", outputPath]
|
|
2495
|
+
);
|
|
2496
|
+
await args.reporter?.onEvent?.({
|
|
2497
|
+
stage: `evaluating_${args.kind}`,
|
|
2498
|
+
status: "running",
|
|
2499
|
+
message: `Starting ${args.kind} Transformers inference.`,
|
|
2500
|
+
details: {
|
|
2501
|
+
model_id: args.modelId,
|
|
2502
|
+
examples: args.examples.length,
|
|
2503
|
+
command: entrypoint.displayCommand,
|
|
2504
|
+
log_path: logPath
|
|
2505
|
+
}
|
|
2506
|
+
});
|
|
2507
|
+
const result = await runLoggedProcess({
|
|
2508
|
+
command: entrypoint.command,
|
|
2509
|
+
commandArgs: entrypoint.commandArgs,
|
|
2510
|
+
env: withBundledPythonEnvironment(
|
|
2511
|
+
withOfflineHuggingFaceCacheEnvironment(
|
|
2512
|
+
minimalMachineLearningEnvironment(process.env),
|
|
2513
|
+
args.config.paths.modelCache
|
|
2514
|
+
)
|
|
2515
|
+
),
|
|
2516
|
+
logPath,
|
|
2517
|
+
timeoutMs: args.config.evaluation.timeoutMs,
|
|
2518
|
+
timeoutMessage: `Transformers inference timed out after ${args.config.evaluation.timeoutMs}ms`,
|
|
2519
|
+
reporter: args.reporter,
|
|
2520
|
+
stage: `evaluating_${args.kind}`,
|
|
2521
|
+
shouldCancel: args.shouldCancel
|
|
2522
|
+
});
|
|
2523
|
+
if (result.exitCode !== 0) {
|
|
2524
|
+
throw new Error(
|
|
2525
|
+
`Transformers inference exited ${result.exitCode}: ${result.stderr.slice(0, 1e3)}`
|
|
2526
|
+
);
|
|
2527
|
+
}
|
|
2528
|
+
let output;
|
|
2529
|
+
try {
|
|
2530
|
+
output = JSON.parse(await readFile6(outputPath, "utf8"));
|
|
2531
|
+
} catch (error) {
|
|
2532
|
+
throw new Error(
|
|
2533
|
+
`Transformers inference did not write valid JSON: ${error instanceof Error ? error.message : String(error)}`
|
|
2534
|
+
);
|
|
2535
|
+
}
|
|
2536
|
+
return parseBatchInferenceResult(
|
|
2537
|
+
output,
|
|
2538
|
+
args.examples.map((_, index) => String(index))
|
|
2539
|
+
);
|
|
2540
|
+
}
|
|
2541
|
+
function assertEvaluationScoringReady(config) {
|
|
2542
|
+
if (config.evaluation.scoring.mode === "json_fields" && config.evaluation.scoring.fields?.some((field) => !field.trim())) {
|
|
2543
|
+
throw new Error("evaluation.scoring.fields must contain non-empty field names");
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
function baselineCacheKey(args) {
|
|
2547
|
+
return createHash4("sha256").update(JSON.stringify({
|
|
2548
|
+
v: 6,
|
|
2549
|
+
inference_protocol_version: INFERENCE_PROTOCOL_VERSION,
|
|
2550
|
+
package_version: args.packageVersion,
|
|
2551
|
+
model_id: args.modelId,
|
|
2552
|
+
base_model_revision: args.baseModelRevision ?? null,
|
|
2553
|
+
source_fingerprint: args.sourceFingerprint ?? null,
|
|
2554
|
+
system: args.system,
|
|
2555
|
+
examples: args.examples,
|
|
2556
|
+
eval_examples_total: args.evalExamplesTotal,
|
|
2557
|
+
eval_split: args.evalSplit ?? null,
|
|
2558
|
+
eval_sample_seed: args.evalSampleSeed ?? null,
|
|
2559
|
+
inference: args.config.evaluation.inference,
|
|
2560
|
+
scoring: args.config.evaluation.scoring
|
|
2561
|
+
})).digest("hex");
|
|
2562
|
+
}
|
|
2563
|
+
function baselineInputsAreStable(args) {
|
|
2564
|
+
return args.config.paths.baseModel ? Boolean(args.sourceFingerprint) : Boolean(args.baseModelRevision);
|
|
2565
|
+
}
|
|
2566
|
+
function baselineCachePath(config, key) {
|
|
2567
|
+
const root = config.storeRoot ? resolve9(config.storeRoot) : defaultLocalHome();
|
|
2568
|
+
return join6(root, "cache", "baseline-evals", `${key}.json`);
|
|
2569
|
+
}
|
|
2570
|
+
async function readBaselineCache(path) {
|
|
2571
|
+
try {
|
|
2572
|
+
return evalReportSchema.parse(JSON.parse(await readFile6(path, "utf8")));
|
|
2573
|
+
} catch {
|
|
2574
|
+
return null;
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
var cachedPackageVersion = null;
|
|
2578
|
+
async function packageVersion() {
|
|
2579
|
+
if (cachedPackageVersion !== null) return cachedPackageVersion;
|
|
2580
|
+
try {
|
|
2581
|
+
const raw = await readFile6(resolve9(localRuntimePackageRoot(import.meta.url), "package.json"), "utf8");
|
|
2582
|
+
cachedPackageVersion = String(
|
|
2583
|
+
JSON.parse(raw).version ?? "unknown"
|
|
2584
|
+
);
|
|
2585
|
+
} catch {
|
|
2586
|
+
cachedPackageVersion = "unknown";
|
|
2587
|
+
}
|
|
2588
|
+
return cachedPackageVersion;
|
|
2589
|
+
}
|
|
2590
|
+
async function scoreGenerated(args) {
|
|
2591
|
+
const jsonScores = [];
|
|
2592
|
+
const results = args.generated.map((generated) => {
|
|
2593
|
+
const json = args.config.evaluation.scoring.mode === "json_fields" ? scoreJsonFields(
|
|
2594
|
+
generated.expected,
|
|
2595
|
+
generated.actual,
|
|
2596
|
+
args.config.evaluation.scoring.fields
|
|
2597
|
+
) : null;
|
|
2598
|
+
if (json) jsonScores.push(json);
|
|
2599
|
+
const score = json?.score ?? exactScore(generated.expected, generated.actual);
|
|
2600
|
+
return {
|
|
2601
|
+
prompt: generated.prompt,
|
|
2602
|
+
expected: generated.expected,
|
|
2603
|
+
actual: generated.actual,
|
|
2604
|
+
passed: json?.passed ?? score === 1,
|
|
2605
|
+
score,
|
|
2606
|
+
reasoning: json?.reasoning ?? (args.inferenceProvider === "none" ? "Dry run: model inference was not executed." : "Scored by normalized exact match."),
|
|
2607
|
+
latency_ms: generated.latency_ms,
|
|
2608
|
+
scored_by: json ? "json_fields" : args.inferenceProvider === "none" ? "heuristic" : "exact_match"
|
|
2609
|
+
};
|
|
2610
|
+
});
|
|
2611
|
+
const total = results.length;
|
|
2612
|
+
const mean = (values) => values.length > 0 ? values.reduce((sum, value) => sum + value, 0) / values.length : 0;
|
|
2613
|
+
const report = evalReportSchema.parse({
|
|
2614
|
+
kind: args.kind,
|
|
2615
|
+
model_id: args.modelId,
|
|
2616
|
+
total,
|
|
2617
|
+
eval_examples_total: args.evalExamplesTotal,
|
|
2618
|
+
eval_examples_used: total,
|
|
2619
|
+
eval_truncated: args.evalExamplesTotal > total,
|
|
2620
|
+
eval_split: args.evalSplit,
|
|
2621
|
+
eval_sample_seed: args.evalSampleSeed ?? null,
|
|
2622
|
+
avg_score: mean(results.map((result) => result.score)),
|
|
2623
|
+
pass_rate: mean(results.map((result) => result.passed ? 1 : 0)),
|
|
2624
|
+
exact_match_rate: mean(
|
|
2625
|
+
results.map((result) => exactScore(result.expected, result.actual))
|
|
2626
|
+
),
|
|
2627
|
+
avg_token_f1: mean(
|
|
2628
|
+
results.map((result) => tokenF1(result.expected, result.actual))
|
|
2629
|
+
),
|
|
2630
|
+
avg_latency_ms: Math.round(mean(results.map((result) => result.latency_ms))),
|
|
2631
|
+
results,
|
|
2632
|
+
artifact_uri: fileUri(args.outputPath),
|
|
2633
|
+
scoring_method: args.config.evaluation.scoring.mode === "json_fields" ? "json_fields" : args.inferenceProvider === "none" ? "heuristic" : "exact_match",
|
|
2634
|
+
inference_provider: args.inferenceProvider,
|
|
2635
|
+
scoring_mode: args.config.evaluation.scoring.mode,
|
|
2636
|
+
json_field_metrics: args.config.evaluation.scoring.mode === "json_fields" ? aggregateJsonFieldMetrics(jsonScores, total) : void 0,
|
|
2637
|
+
generation_config: args.generationConfig,
|
|
2638
|
+
log_uri: args.logUri
|
|
2639
|
+
});
|
|
2640
|
+
await writeJson(args.outputPath, report);
|
|
2641
|
+
return report;
|
|
2642
|
+
}
|
|
2643
|
+
async function evaluateExamples(args) {
|
|
2644
|
+
assertEvaluationScoringReady(args.config);
|
|
2645
|
+
const maxExamples = args.config.evaluation.maxExamples ?? args.maxExamples ?? args.examples.length;
|
|
2646
|
+
const truncated = args.examples.length > maxExamples;
|
|
2647
|
+
const seed = args.config.evaluation.sampleSeed ?? args.sampleSeed ?? 0;
|
|
2648
|
+
const examples = truncated ? sampleExamples(args.examples, maxExamples, seed) : args.examples;
|
|
2649
|
+
const evalSampleSeed = truncated || args.evalSplit === "spec_holdout" ? seed : null;
|
|
2650
|
+
const cacheEligible = args.kind === "baseline" && args.config.evaluation.baselineCache && !args.config.dryRun && baselineInputsAreStable({
|
|
2651
|
+
baseModelRevision: args.baseModelRevision,
|
|
2652
|
+
sourceFingerprint: args.sourceFingerprint,
|
|
2653
|
+
config: args.config
|
|
2654
|
+
});
|
|
2655
|
+
const cacheKey = cacheEligible ? baselineCacheKey({
|
|
2656
|
+
modelId: args.modelId,
|
|
2657
|
+
baseModelRevision: args.baseModelRevision,
|
|
2658
|
+
sourceFingerprint: args.sourceFingerprint,
|
|
2659
|
+
system: args.system,
|
|
2660
|
+
examples,
|
|
2661
|
+
evalExamplesTotal: args.examples.length,
|
|
2662
|
+
evalSplit: args.evalSplit,
|
|
2663
|
+
evalSampleSeed,
|
|
2664
|
+
config: args.config,
|
|
2665
|
+
packageVersion: await packageVersion()
|
|
2666
|
+
}) : null;
|
|
2667
|
+
if (cacheKey) {
|
|
2668
|
+
const cached = await readBaselineCache(baselineCachePath(args.config, cacheKey));
|
|
2669
|
+
if (cached) {
|
|
2670
|
+
const report2 = evalReportSchema.parse({
|
|
2671
|
+
...cached,
|
|
2672
|
+
cached: true,
|
|
2673
|
+
cache_key: cacheKey,
|
|
2674
|
+
artifact_uri: fileUri(args.outputPath),
|
|
2675
|
+
eval_split: args.evalSplit ?? cached.eval_split,
|
|
2676
|
+
eval_examples_total: args.examples.length,
|
|
2677
|
+
eval_examples_used: examples.length,
|
|
2678
|
+
eval_truncated: truncated,
|
|
2679
|
+
eval_sample_seed: evalSampleSeed
|
|
2680
|
+
});
|
|
2681
|
+
await writeJson(args.outputPath, report2);
|
|
2682
|
+
return report2;
|
|
2683
|
+
}
|
|
2684
|
+
}
|
|
2685
|
+
const inference = args.config.dryRun ? null : await runTransformersInference({
|
|
2686
|
+
kind: args.kind,
|
|
2687
|
+
modelId: args.modelId,
|
|
2688
|
+
baseModelId: args.baseModelId ?? args.modelId,
|
|
2689
|
+
baseModelRevision: args.baseModelRevision,
|
|
2690
|
+
adapterPath: args.adapterPath,
|
|
2691
|
+
examples,
|
|
2692
|
+
system: args.system,
|
|
2693
|
+
config: args.config,
|
|
2694
|
+
outputPath: args.outputPath,
|
|
2695
|
+
reporter: args.reporter,
|
|
2696
|
+
shouldCancel: args.shouldCancel
|
|
2697
|
+
});
|
|
2698
|
+
const generated = examples.map((example, index) => ({
|
|
2699
|
+
prompt: example.input,
|
|
2700
|
+
expected: example.output,
|
|
2701
|
+
actual: inference?.results[index]?.actual ?? "",
|
|
2702
|
+
latency_ms: inference?.results[index]?.latency_ms ?? 0
|
|
2703
|
+
}));
|
|
2704
|
+
const report = await scoreGenerated({
|
|
2705
|
+
kind: args.kind,
|
|
2706
|
+
modelId: args.modelId,
|
|
2707
|
+
generated,
|
|
2708
|
+
evalExamplesTotal: args.examples.length,
|
|
2709
|
+
config: args.config,
|
|
2710
|
+
outputPath: args.outputPath,
|
|
2711
|
+
inferenceProvider: inference ? "transformers" : "none",
|
|
2712
|
+
evalSplit: args.evalSplit,
|
|
2713
|
+
evalSampleSeed,
|
|
2714
|
+
generationConfig: inference?.generation_config,
|
|
2715
|
+
logUri: inference ? fileUri(`${args.outputPath}.inference.log`) : void 0
|
|
2716
|
+
});
|
|
2717
|
+
if (cacheKey) {
|
|
2718
|
+
await writeJson(
|
|
2719
|
+
baselineCachePath(args.config, cacheKey),
|
|
2720
|
+
{ ...report, cache_key: cacheKey }
|
|
2721
|
+
);
|
|
2722
|
+
}
|
|
2723
|
+
return report;
|
|
2724
|
+
}
|
|
2725
|
+
function compareEvalReports(baseline, candidate) {
|
|
2726
|
+
let regressions = 0;
|
|
2727
|
+
let improvements = 0;
|
|
2728
|
+
const regressedExamples = [];
|
|
2729
|
+
const count = Math.min(baseline.results.length, candidate.results.length);
|
|
2730
|
+
for (let index = 0; index < count; index += 1) {
|
|
2731
|
+
const oldScore = baseline.results[index]?.score ?? 0;
|
|
2732
|
+
const result = candidate.results[index];
|
|
2733
|
+
const newScore = result?.score ?? 0;
|
|
2734
|
+
if (newScore < oldScore) {
|
|
2735
|
+
regressions += 1;
|
|
2736
|
+
regressedExamples.push({
|
|
2737
|
+
prompt: baseline.results[index]?.prompt ?? "",
|
|
2738
|
+
old_score: oldScore,
|
|
2739
|
+
new_score: newScore
|
|
2740
|
+
});
|
|
2741
|
+
} else if (newScore > oldScore) {
|
|
2742
|
+
improvements += 1;
|
|
2743
|
+
}
|
|
2744
|
+
}
|
|
2745
|
+
return {
|
|
2746
|
+
avg_score_delta: candidate.avg_score - baseline.avg_score,
|
|
2747
|
+
pass_rate_delta: candidate.pass_rate - baseline.pass_rate,
|
|
2748
|
+
exact_match_rate_delta: candidate.exact_match_rate - baseline.exact_match_rate,
|
|
2749
|
+
token_f1_delta: (candidate.avg_token_f1 ?? 0) - (baseline.avg_token_f1 ?? 0),
|
|
2750
|
+
regressions,
|
|
2751
|
+
improvements,
|
|
2752
|
+
regressed_examples: regressedExamples
|
|
2753
|
+
};
|
|
2754
|
+
}
|
|
2755
|
+
|
|
2756
|
+
// src/local-runtime/process-training.ts
|
|
2757
|
+
import { access, mkdir as mkdir7, readFile as readFile7, writeFile as writeFile5 } from "fs/promises";
|
|
2758
|
+
import { join as join7, resolve as resolve10 } from "path";
|
|
2759
|
+
function buildTrainingHyperparameters(request, options = {}) {
|
|
2760
|
+
const model = resolveTrainingModel(request.spec_snapshot.base_model);
|
|
2761
|
+
const hyper = request.hyperparameters;
|
|
2762
|
+
return {
|
|
2763
|
+
base_model: model.id,
|
|
2764
|
+
...options.baseModelRevision ? { base_model_revision: options.baseModelRevision } : {},
|
|
2765
|
+
n_epochs: String(hyper.n_epochs),
|
|
2766
|
+
learning_rate: String(hyper.learning_rate ?? model.defaultLearningRate),
|
|
2767
|
+
per_device_train_batch_size: String(hyper.batch_size ?? model.defaultPerDeviceBatchSize),
|
|
2768
|
+
gradient_accumulation_steps: String(
|
|
2769
|
+
hyper.gradient_accumulation_steps ?? model.defaultGradientAccumulationSteps
|
|
2770
|
+
),
|
|
2771
|
+
lora_rank: String(hyper.lora_rank ?? model.defaultLoraRank),
|
|
2772
|
+
lora_alpha: String(hyper.lora_alpha ?? model.defaultLoraAlpha),
|
|
2773
|
+
lora_dropout: String(hyper.lora_dropout ?? model.defaultLoraDropout),
|
|
2774
|
+
max_seq_length: String(hyper.max_seq_length ?? model.defaultMaxSeqLength)
|
|
2775
|
+
};
|
|
2776
|
+
}
|
|
2777
|
+
async function pathExists(path) {
|
|
2778
|
+
try {
|
|
2779
|
+
await access(path);
|
|
2780
|
+
return true;
|
|
2781
|
+
} catch {
|
|
2782
|
+
return false;
|
|
2783
|
+
}
|
|
2784
|
+
}
|
|
2785
|
+
function numberFromMetric(value) {
|
|
2786
|
+
if (!value) return void 0;
|
|
2787
|
+
const parsed = Number(value);
|
|
2788
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
2789
|
+
}
|
|
2790
|
+
function parseTrainingProgressLine(line) {
|
|
2791
|
+
if (/loading (?:checkpoint )?shards|loading weights/i.test(line)) return null;
|
|
2792
|
+
const snapshot = {};
|
|
2793
|
+
const metricMatches = [...line.matchAll(/'([^']+)':\s*'([^']*)'/g)];
|
|
2794
|
+
for (const match of metricMatches) {
|
|
2795
|
+
const key = match[1];
|
|
2796
|
+
const value = numberFromMetric(match[2]);
|
|
2797
|
+
if (value === void 0) continue;
|
|
2798
|
+
if (key === "loss") snapshot.loss = value;
|
|
2799
|
+
if (key === "grad_norm") snapshot.grad_norm = value;
|
|
2800
|
+
if (key === "learning_rate") snapshot.learning_rate = value;
|
|
2801
|
+
if (key === "epoch") {
|
|
2802
|
+
snapshot.epoch = value;
|
|
2803
|
+
snapshot.percent = Math.max(0, Math.min(100, Math.round(value * 100)));
|
|
2804
|
+
}
|
|
2805
|
+
}
|
|
2806
|
+
const progress = line.match(/(\d+)%\|[^|]*\|\s*(\d+)\/(\d+)\s*\[([^<,\]]+)(?:<([^,\]]+))?,\s*([^\]]+)\]/);
|
|
2807
|
+
if (progress) {
|
|
2808
|
+
snapshot.percent = Number(progress[1]);
|
|
2809
|
+
snapshot.step = Number(progress[2]);
|
|
2810
|
+
snapshot.total_steps = Number(progress[3]);
|
|
2811
|
+
snapshot.elapsed = progress[4]?.trim();
|
|
2812
|
+
snapshot.eta = progress[5]?.trim();
|
|
2813
|
+
snapshot.rate = progress[6]?.trim();
|
|
2814
|
+
}
|
|
2815
|
+
return Object.keys(snapshot).length > 0 ? snapshot : null;
|
|
2816
|
+
}
|
|
2817
|
+
function createTrainingProgressForwarder(reporter) {
|
|
2818
|
+
let lastPercent = -1;
|
|
2819
|
+
let lastStep = -1;
|
|
2820
|
+
let lastEmittedAt = 0;
|
|
2821
|
+
let latestMetrics = {};
|
|
2822
|
+
return (line) => {
|
|
2823
|
+
const parsed = parseTrainingProgressLine(line);
|
|
2824
|
+
if (!parsed) return;
|
|
2825
|
+
latestMetrics = { ...latestMetrics, ...parsed };
|
|
2826
|
+
const percent = latestMetrics.percent;
|
|
2827
|
+
const step = latestMetrics.step;
|
|
2828
|
+
const totalSteps = latestMetrics.total_steps;
|
|
2829
|
+
const now = Date.now();
|
|
2830
|
+
const shouldEmit = Boolean(reporter?.onEvent) && (percent !== void 0 && (percent >= lastPercent + 5 || percent === 100) || step !== void 0 && totalSteps !== void 0 && step === totalSteps && step !== lastStep || now - lastEmittedAt > 3e4);
|
|
2831
|
+
if (!shouldEmit) return;
|
|
2832
|
+
lastEmittedAt = now;
|
|
2833
|
+
if (percent !== void 0) lastPercent = percent;
|
|
2834
|
+
if (step !== void 0) lastStep = step;
|
|
2835
|
+
reportInBackground(() => reporter?.onEvent?.({
|
|
2836
|
+
stage: "training",
|
|
2837
|
+
status: "running",
|
|
2838
|
+
message: step !== void 0 && totalSteps !== void 0 ? `Training progress ${step}/${totalSteps}${percent !== void 0 ? ` (${percent}%)` : ""}.` : `Training progress${percent !== void 0 ? ` ${percent}%` : ""}.`,
|
|
2839
|
+
details: latestMetrics
|
|
2840
|
+
}));
|
|
2841
|
+
};
|
|
2842
|
+
}
|
|
2843
|
+
async function launchProcessTraining(args) {
|
|
2844
|
+
const { request, artifacts, config } = args;
|
|
2845
|
+
const jobName = `tt-local-${request.run_id}`;
|
|
2846
|
+
await mkdir7(artifacts.trainingDir, { recursive: true });
|
|
2847
|
+
await copyDatasetToTrainingChannel(artifacts);
|
|
2848
|
+
const hyperparameters = buildTrainingHyperparameters(request, {
|
|
2849
|
+
baseModelRevision: args.baseModelRevision
|
|
2850
|
+
});
|
|
2851
|
+
await writeJson(join7(artifacts.trainingConfigDir, "hyperparameters.json"), hyperparameters);
|
|
2852
|
+
const entrypoint = buildBundledPythonCommand("train.py");
|
|
2853
|
+
if (config.dryRun) {
|
|
2854
|
+
await writeFile5(
|
|
2855
|
+
artifacts.trainingLog,
|
|
2856
|
+
"Dry run enabled. Training process was not launched.\n",
|
|
2857
|
+
"utf8"
|
|
2858
|
+
);
|
|
2859
|
+
return {
|
|
2860
|
+
provider: "local-uv",
|
|
2861
|
+
training_job_name: jobName,
|
|
2862
|
+
model_artifact_uri: fileUri(artifacts.trainingModelDir),
|
|
2863
|
+
base_model_artifact_uri: config.paths.baseModel ? fileUri(config.paths.baseModel) : void 0,
|
|
2864
|
+
metrics: { dry_run: true },
|
|
2865
|
+
exit_code: 0,
|
|
2866
|
+
log_uri: fileUri(artifacts.trainingLog),
|
|
2867
|
+
command: entrypoint.displayCommand
|
|
2868
|
+
};
|
|
2869
|
+
}
|
|
2870
|
+
const command = entrypoint.displayCommand;
|
|
2871
|
+
const inheritedEnv = minimalMachineLearningEnvironment(process.env);
|
|
2872
|
+
const childEnv = withBundledPythonEnvironment(
|
|
2873
|
+
withOfflineHuggingFaceCacheEnvironment({
|
|
2874
|
+
...inheritedEnv,
|
|
2875
|
+
BACKEND: "local",
|
|
2876
|
+
SM_CHANNEL_TRAINING: resolve10(artifacts.trainingInputDir),
|
|
2877
|
+
SM_CHANNEL_BASE_MODEL: config.paths.baseModel ? resolve10(config.paths.baseModel) : void 0,
|
|
2878
|
+
SM_MODEL_DIR: resolve10(artifacts.trainingModelDir),
|
|
2879
|
+
SM_OUTPUT_DIR: resolve10(artifacts.trainingOutputDir),
|
|
2880
|
+
TT_HYPERPARAMETERS_PATH: resolve10(artifacts.trainingConfigDir, "hyperparameters.json")
|
|
2881
|
+
}, config.paths.modelCache)
|
|
2882
|
+
);
|
|
2883
|
+
for (const [key, value] of Object.entries(childEnv)) {
|
|
2884
|
+
if (value === void 0) delete childEnv[key];
|
|
2885
|
+
}
|
|
2886
|
+
if (config.paths.modelCache) await mkdir7(resolve10(config.paths.modelCache), { recursive: true });
|
|
2887
|
+
await args.reporter?.onEvent?.({
|
|
2888
|
+
stage: "training",
|
|
2889
|
+
status: "running",
|
|
2890
|
+
message: "Starting local training process.",
|
|
2891
|
+
details: {
|
|
2892
|
+
command,
|
|
2893
|
+
log_path: artifacts.trainingLog,
|
|
2894
|
+
model_dir: artifacts.trainingModelDir,
|
|
2895
|
+
output_dir: artifacts.trainingOutputDir
|
|
2896
|
+
}
|
|
2897
|
+
});
|
|
2898
|
+
const forwardTrainingProgress = createTrainingProgressForwarder(args.reporter);
|
|
2899
|
+
const { exitCode } = await runLoggedProcess({
|
|
2900
|
+
command: entrypoint.command,
|
|
2901
|
+
commandArgs: entrypoint.commandArgs,
|
|
2902
|
+
env: childEnv,
|
|
2903
|
+
logPath: artifacts.trainingLog,
|
|
2904
|
+
reporter: args.reporter,
|
|
2905
|
+
stage: "training",
|
|
2906
|
+
onLine: (line) => forwardTrainingProgress(line),
|
|
2907
|
+
shouldCancel: args.shouldCancel
|
|
2908
|
+
});
|
|
2909
|
+
const metricsPath = join7(artifacts.trainingModelDir, "training-metrics.json");
|
|
2910
|
+
const metrics = await pathExists(metricsPath) ? JSON.parse(await readFile7(metricsPath, "utf8")) : null;
|
|
2911
|
+
if (exitCode !== 0) {
|
|
2912
|
+
throw new Error(`Training process exited with code ${exitCode}. See ${artifacts.trainingLog}.`);
|
|
2913
|
+
}
|
|
2914
|
+
const modelTarPath = join7(artifacts.trainingOutputDir, "model.tar.gz");
|
|
2915
|
+
const modelUri = await pathExists(modelTarPath) ? fileUri(modelTarPath) : fileUri(artifacts.trainingModelDir);
|
|
2916
|
+
await args.reporter?.onEvent?.({
|
|
2917
|
+
stage: "training",
|
|
2918
|
+
status: "running",
|
|
2919
|
+
message: "Training process completed.",
|
|
2920
|
+
details: {
|
|
2921
|
+
exit_code: exitCode,
|
|
2922
|
+
model_artifact_uri: modelUri,
|
|
2923
|
+
log_path: artifacts.trainingLog,
|
|
2924
|
+
metrics
|
|
2925
|
+
}
|
|
2926
|
+
});
|
|
2927
|
+
return {
|
|
2928
|
+
provider: "local-uv",
|
|
2929
|
+
training_job_name: jobName,
|
|
2930
|
+
model_artifact_uri: modelUri,
|
|
2931
|
+
base_model_artifact_uri: config.paths.baseModel ? fileUri(config.paths.baseModel) : void 0,
|
|
2932
|
+
metrics,
|
|
2933
|
+
exit_code: exitCode,
|
|
2934
|
+
log_uri: fileUri(artifacts.trainingLog),
|
|
2935
|
+
command
|
|
2936
|
+
};
|
|
2937
|
+
}
|
|
2938
|
+
|
|
2939
|
+
// src/local-runtime/orchestrator.ts
|
|
2940
|
+
function canonicalLocalPipeline() {
|
|
2941
|
+
return {
|
|
2942
|
+
version: 1,
|
|
2943
|
+
name: "local-fine-tune",
|
|
2944
|
+
steps: [
|
|
2945
|
+
{ id: "baseline", uses: "evaluate", target: "local", with: { model: "base", evaluator: "behavior" } },
|
|
2946
|
+
{ id: "train", uses: "train", target: "local" },
|
|
2947
|
+
{ id: "candidate", uses: "evaluate", target: "local", with: { model: { from: "train.model" }, evaluator: "behavior" } },
|
|
2948
|
+
{ id: "compare", uses: "compare", target: "local", with: { before: { from: "baseline.report" }, after: { from: "candidate.report" } } }
|
|
2949
|
+
]
|
|
2950
|
+
};
|
|
2951
|
+
}
|
|
2952
|
+
function referenceParts(reference) {
|
|
2953
|
+
const value = typeof reference === "string" ? reference.replace(/^\$/, "") : reference && typeof reference === "object" && !Array.isArray(reference) ? reference.from : void 0;
|
|
2954
|
+
if (typeof value !== "string") return null;
|
|
2955
|
+
const match = /^([A-Za-z][A-Za-z0-9_-]*)\.(model|report)$/.exec(value);
|
|
2956
|
+
return match ? [match[1], match[2]] : null;
|
|
2957
|
+
}
|
|
2958
|
+
function validateLocalPipeline(pipeline) {
|
|
2959
|
+
if (!pipeline || pipeline.version !== 1 || !Array.isArray(pipeline.steps)) {
|
|
2960
|
+
throw new Error("Local pipeline must be a version 1 document with a steps array.");
|
|
2961
|
+
}
|
|
2962
|
+
const seen = /* @__PURE__ */ new Map();
|
|
2963
|
+
let hasTrainStep = false;
|
|
2964
|
+
for (const step of pipeline.steps) {
|
|
2965
|
+
if (!step || typeof step.id !== "string" || !/^[A-Za-z][A-Za-z0-9_-]*$/.test(step.id)) {
|
|
2966
|
+
throw new Error("Each local pipeline step requires an identifier starting with a letter.");
|
|
2967
|
+
}
|
|
2968
|
+
if (seen.has(step.id)) throw new Error(`Local pipeline step IDs must be unique: ${step.id}`);
|
|
2969
|
+
if (step.target !== "local") throw new Error(`Local pipeline step ${step.id} only supports target local.`);
|
|
2970
|
+
if (step.uses !== "train" && step.uses !== "evaluate" && step.uses !== "compare") {
|
|
2971
|
+
throw new Error(`Local pipeline step ${step.id} uses an unsupported component.`);
|
|
2972
|
+
}
|
|
2973
|
+
if (step.uses === "train") {
|
|
2974
|
+
if (hasTrainStep) {
|
|
2975
|
+
throw new Error("Local pipeline v1 supports at most one train step because model artifacts are run-scoped.");
|
|
2976
|
+
}
|
|
2977
|
+
hasTrainStep = true;
|
|
2978
|
+
}
|
|
2979
|
+
if (step.uses === "evaluate") {
|
|
2980
|
+
if (step.with?.evaluator !== "behavior") {
|
|
2981
|
+
throw new Error(`Local evaluate step ${step.id} requires the behavior evaluator.`);
|
|
2982
|
+
}
|
|
2983
|
+
if (step.with?.model !== "base") {
|
|
2984
|
+
const modelReference = referenceParts(step.with?.model);
|
|
2985
|
+
if (!modelReference || modelReference[0] === step.id || modelReference[1] !== "model") {
|
|
2986
|
+
throw new Error(`Local evaluate step ${step.id} requires model base or { from: '<earlier-train>.model' }.`);
|
|
2987
|
+
}
|
|
2988
|
+
const producer = seen.get(modelReference[0]);
|
|
2989
|
+
if (!producer) throw new Error(`Local evaluate step ${step.id} must reference an earlier train step.`);
|
|
2990
|
+
if (producer.uses !== "train") throw new Error(`Local evaluate step ${step.id} model reference must name a train step.`);
|
|
2991
|
+
}
|
|
2992
|
+
}
|
|
2993
|
+
if (step.uses === "compare") {
|
|
2994
|
+
for (const field of ["before", "after"]) {
|
|
2995
|
+
const reportReference = referenceParts(step.with?.[field]);
|
|
2996
|
+
if (!reportReference || reportReference[1] !== "report") {
|
|
2997
|
+
throw new Error(`Local compare step ${step.id} requires ${field}: { from: '<earlier-evaluate>.report' }.`);
|
|
2998
|
+
}
|
|
2999
|
+
const producer = seen.get(reportReference[0]);
|
|
3000
|
+
if (!producer) throw new Error(`Local compare step ${step.id} must reference an earlier evaluate step.`);
|
|
3001
|
+
if (producer.uses !== "evaluate") throw new Error(`Local compare step ${step.id} report reference must name an evaluate step.`);
|
|
3002
|
+
}
|
|
3003
|
+
}
|
|
3004
|
+
seen.set(step.id, step);
|
|
3005
|
+
}
|
|
3006
|
+
return pipeline;
|
|
3007
|
+
}
|
|
3008
|
+
async function loadJsonFile(path) {
|
|
3009
|
+
return JSON.parse(await readFile8(path, "utf8"));
|
|
3010
|
+
}
|
|
3011
|
+
async function loadRunRequest(path) {
|
|
3012
|
+
return fineTuneRunRequestSchema.parse(await loadJsonFile(path));
|
|
3013
|
+
}
|
|
3014
|
+
async function loadLocalRunnerConfig(path) {
|
|
3015
|
+
if (!path) return localRunnerConfigSchema.parse({});
|
|
3016
|
+
const configPath = resolve11(path);
|
|
3017
|
+
const base = dirname8(configPath);
|
|
3018
|
+
const config = localRunnerConfigSchema.parse(await loadJsonFile(configPath));
|
|
3019
|
+
const configPathValue = (value) => {
|
|
3020
|
+
if (!value) return void 0;
|
|
3021
|
+
if (value === "~") return homedir5();
|
|
3022
|
+
if (value.startsWith("~/")) return resolve11(homedir5(), value.slice(2));
|
|
3023
|
+
return resolve11(base, value);
|
|
3024
|
+
};
|
|
3025
|
+
return {
|
|
3026
|
+
...config,
|
|
3027
|
+
artifactRoot: configPathValue(config.artifactRoot),
|
|
3028
|
+
storeRoot: configPathValue(config.storeRoot),
|
|
3029
|
+
paths: {
|
|
3030
|
+
baseModel: configPathValue(config.paths.baseModel),
|
|
3031
|
+
modelCache: configPathValue(config.paths.modelCache)
|
|
3032
|
+
},
|
|
3033
|
+
evaluation: {
|
|
3034
|
+
...config.evaluation,
|
|
3035
|
+
generalRegression: config.evaluation.generalRegression ? {
|
|
3036
|
+
...config.evaluation.generalRegression,
|
|
3037
|
+
dataset: configPathValue(config.evaluation.generalRegression.dataset)
|
|
3038
|
+
} : void 0
|
|
3039
|
+
}
|
|
3040
|
+
};
|
|
3041
|
+
}
|
|
3042
|
+
function elapsed(started) {
|
|
3043
|
+
const ms = Math.max(0, Math.round(performance.now() - started));
|
|
3044
|
+
return { ms, seconds: Math.round(ms / 1e3 * 1e3) / 1e3 };
|
|
3045
|
+
}
|
|
3046
|
+
function stripFileUri(path) {
|
|
3047
|
+
return path.replace(/^file:\/\//, "");
|
|
3048
|
+
}
|
|
3049
|
+
async function pathExists2(path) {
|
|
3050
|
+
try {
|
|
3051
|
+
await stat5(path);
|
|
3052
|
+
return true;
|
|
3053
|
+
} catch {
|
|
3054
|
+
return false;
|
|
3055
|
+
}
|
|
3056
|
+
}
|
|
3057
|
+
function selectPrebuiltEvaluationSplit(dataset) {
|
|
3058
|
+
if (!dataset) throw new Error("dataset_prebuilt is required");
|
|
3059
|
+
if (dataset.validation) return "prebuilt_validation";
|
|
3060
|
+
if (dataset.test) return "prebuilt_test";
|
|
3061
|
+
throw new Error("dataset_prebuilt requires a distinct validation or test split");
|
|
3062
|
+
}
|
|
3063
|
+
function assertMatchingDatasetSystems(jsonl, expectedSystem, split) {
|
|
3064
|
+
for (const [index, line] of jsonl.split(/\r?\n/).entries()) {
|
|
3065
|
+
if (!line.trim()) continue;
|
|
3066
|
+
const row = JSON.parse(line);
|
|
3067
|
+
const actualSystem = row.messages.find((message) => message.role === "system")?.content ?? "";
|
|
3068
|
+
if (actualSystem !== expectedSystem) {
|
|
3069
|
+
throw new Error(
|
|
3070
|
+
`Prebuilt ${split} row ${index + 1} system message must match the behavior spec system message.`
|
|
3071
|
+
);
|
|
3072
|
+
}
|
|
3073
|
+
}
|
|
3074
|
+
}
|
|
3075
|
+
function addDryRunPlaceholders(request, dryRun) {
|
|
3076
|
+
if (!dryRun || !request || typeof request !== "object" || Array.isArray(request)) return request;
|
|
3077
|
+
const candidate = request;
|
|
3078
|
+
if (candidate.dataset_prebuilt) return request;
|
|
3079
|
+
const snapshot = candidate.spec_snapshot;
|
|
3080
|
+
if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)) return request;
|
|
3081
|
+
const examples = snapshot.examples;
|
|
3082
|
+
if (!Array.isArray(examples) || examples.length > 0) return request;
|
|
3083
|
+
return {
|
|
3084
|
+
...candidate,
|
|
3085
|
+
spec_snapshot: {
|
|
3086
|
+
...snapshot,
|
|
3087
|
+
examples: [
|
|
3088
|
+
{ input: "Dry-run placeholder input A", output: "Dry-run placeholder output A" },
|
|
3089
|
+
{ input: "Dry-run placeholder input B", output: "Dry-run placeholder output B" }
|
|
3090
|
+
]
|
|
3091
|
+
}
|
|
3092
|
+
};
|
|
3093
|
+
}
|
|
3094
|
+
async function validateDatasetInputs(request, config) {
|
|
3095
|
+
const inputIdentity = (value) => value.normalize("NFKC").trim().replace(/\s+/g, " ").toLocaleLowerCase("en");
|
|
3096
|
+
const evalSampleSeed = config.evaluation.sampleSeed ?? deriveSampleSeed(request.behavior_spec_id);
|
|
3097
|
+
if (!request.dataset_prebuilt) {
|
|
3098
|
+
const inlineExamples = examplesFromSpec(request.spec_snapshot);
|
|
3099
|
+
const seenInputs = /* @__PURE__ */ new Map();
|
|
3100
|
+
for (const [index, example] of inlineExamples.entries()) {
|
|
3101
|
+
const identity = inputIdentity(example.input);
|
|
3102
|
+
const previous = seenInputs.get(identity);
|
|
3103
|
+
if (previous !== void 0) {
|
|
3104
|
+
throw new Error(
|
|
3105
|
+
`Inline examples ${previous + 1} and ${index + 1} have duplicate inputs; training and held-out evaluation require distinct prompts.`
|
|
3106
|
+
);
|
|
3107
|
+
}
|
|
3108
|
+
seenInputs.set(identity, index);
|
|
3109
|
+
}
|
|
3110
|
+
if (!config.dryRun && inlineExamples.length < 2) {
|
|
3111
|
+
throw new Error(
|
|
3112
|
+
"A real local fine-tune requires at least 2 inline examples so training and evaluation use distinct data."
|
|
3113
|
+
);
|
|
3114
|
+
}
|
|
3115
|
+
const split = splitSpecExamples(inlineExamples, evalSampleSeed);
|
|
3116
|
+
const hasHoldout = split.holdout.length > 0;
|
|
3117
|
+
const trainingExamples = hasHoldout ? split.train : inlineExamples;
|
|
3118
|
+
const evaluationExamples = hasHoldout ? split.holdout : inlineExamples;
|
|
3119
|
+
return {
|
|
3120
|
+
trainingJsonl: compileSpecToJsonl({ ...request.spec_snapshot, examples: trainingExamples }),
|
|
3121
|
+
trainingExampleCount: trainingExamples.length,
|
|
3122
|
+
evaluationExamples,
|
|
3123
|
+
evalSplit: hasHoldout ? "spec_holdout" : "spec_examples"
|
|
3124
|
+
};
|
|
3125
|
+
}
|
|
3126
|
+
const dataset = request.dataset_prebuilt;
|
|
3127
|
+
const splitEntries = [
|
|
3128
|
+
["training", dataset.training],
|
|
3129
|
+
["validation", dataset.validation],
|
|
3130
|
+
["test", dataset.test]
|
|
3131
|
+
];
|
|
3132
|
+
const normalized = /* @__PURE__ */ new Map();
|
|
3133
|
+
const expectedSystem = buildSystemMessage(request.spec_snapshot);
|
|
3134
|
+
for (const [name, value] of splitEntries) {
|
|
3135
|
+
if (!value) continue;
|
|
3136
|
+
const path = stripFileUri(value);
|
|
3137
|
+
const jsonl = await normalizeChatJsonlForRelocation(path);
|
|
3138
|
+
assertMatchingDatasetSystems(jsonl, expectedSystem, name);
|
|
3139
|
+
normalized.set(name, {
|
|
3140
|
+
jsonl,
|
|
3141
|
+
examples: await examplesFromChatJsonl(path)
|
|
3142
|
+
});
|
|
3143
|
+
}
|
|
3144
|
+
const evaluationSplit = selectPrebuiltEvaluationSplit(dataset);
|
|
3145
|
+
const evaluationKey = evaluationSplit === "prebuilt_validation" ? "validation" : "test";
|
|
3146
|
+
const training = normalized.get("training");
|
|
3147
|
+
const evaluationData = normalized.get(evaluationKey);
|
|
3148
|
+
if (!training || !evaluationData) {
|
|
3149
|
+
throw new Error("Unable to load the configured prebuilt training and evaluation datasets.");
|
|
3150
|
+
}
|
|
3151
|
+
const trainingInputs = new Set(
|
|
3152
|
+
training.examples.map((example) => inputIdentity(example.input))
|
|
3153
|
+
);
|
|
3154
|
+
for (const splitName of ["validation", "test"]) {
|
|
3155
|
+
const split = normalized.get(splitName);
|
|
3156
|
+
if (!split) continue;
|
|
3157
|
+
const splitInputs = /* @__PURE__ */ new Set();
|
|
3158
|
+
for (const example of split.examples) {
|
|
3159
|
+
const identity = inputIdentity(example.input);
|
|
3160
|
+
if (splitInputs.has(identity)) {
|
|
3161
|
+
throw new Error(
|
|
3162
|
+
`Prebuilt ${splitName} data contains duplicate inputs; evaluation prompts must be unique so metrics and comparisons remain well-defined.`
|
|
3163
|
+
);
|
|
3164
|
+
}
|
|
3165
|
+
splitInputs.add(identity);
|
|
3166
|
+
}
|
|
3167
|
+
const overlap = split.examples.filter(
|
|
3168
|
+
(example) => trainingInputs.has(inputIdentity(example.input))
|
|
3169
|
+
);
|
|
3170
|
+
if (overlap.length > 0) {
|
|
3171
|
+
throw new Error(
|
|
3172
|
+
`Prebuilt training and ${splitName} data overlap on ${overlap.length} input(s); evaluation examples must be held out from training.`
|
|
3173
|
+
);
|
|
3174
|
+
}
|
|
3175
|
+
}
|
|
3176
|
+
return {
|
|
3177
|
+
trainingJsonl: training.jsonl,
|
|
3178
|
+
trainingExampleCount: training.examples.length,
|
|
3179
|
+
evaluationExamples: evaluationData.examples,
|
|
3180
|
+
evalSplit: evaluationSplit
|
|
3181
|
+
};
|
|
3182
|
+
}
|
|
3183
|
+
async function validateLocalFineTuneInput(input) {
|
|
3184
|
+
const config = localRunnerConfigSchema.parse(input.config);
|
|
3185
|
+
const request = fineTuneRunRequestSchema.parse(addDryRunPlaceholders(input.request, config.dryRun));
|
|
3186
|
+
await validateDatasetInputs(request, config);
|
|
3187
|
+
if (config.evaluation.generalRegression) {
|
|
3188
|
+
await evaluationSuiteFromChatJsonl(
|
|
3189
|
+
config.evaluation.generalRegression.dataset,
|
|
3190
|
+
config.evaluation.generalRegression.systemPrompt
|
|
3191
|
+
);
|
|
3192
|
+
}
|
|
3193
|
+
return { request, config };
|
|
3194
|
+
}
|
|
3195
|
+
function artifactPrefix(request) {
|
|
3196
|
+
return defaultArtifactPrefix({
|
|
3197
|
+
userId: request.user_id,
|
|
3198
|
+
behaviorSpecId: request.behavior_spec_id,
|
|
3199
|
+
runId: request.run_id
|
|
3200
|
+
});
|
|
3201
|
+
}
|
|
3202
|
+
function canonicalJson2(value) {
|
|
3203
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null";
|
|
3204
|
+
if (Array.isArray(value)) return `[${value.map((entry) => canonicalJson2(entry)).join(",")}]`;
|
|
3205
|
+
const object = value;
|
|
3206
|
+
return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(object[key])}`).join(",")}}`;
|
|
3207
|
+
}
|
|
3208
|
+
function hashJson(value) {
|
|
3209
|
+
return createHash5("sha256").update(canonicalJson2(value)).digest("hex");
|
|
3210
|
+
}
|
|
3211
|
+
var packageRoot2 = localRuntimePackageRoot(import.meta.url);
|
|
3212
|
+
async function hashFileIfPresent(path) {
|
|
3213
|
+
try {
|
|
3214
|
+
return await hashFile(path);
|
|
3215
|
+
} catch {
|
|
3216
|
+
return null;
|
|
3217
|
+
}
|
|
3218
|
+
}
|
|
3219
|
+
async function packageVersion2() {
|
|
3220
|
+
try {
|
|
3221
|
+
const parsed = JSON.parse(await readFile8(resolve11(packageRoot2, "package.json"), "utf8"));
|
|
3222
|
+
return typeof parsed.version === "string" ? parsed.version : "unknown";
|
|
3223
|
+
} catch {
|
|
3224
|
+
return "unknown";
|
|
3225
|
+
}
|
|
3226
|
+
}
|
|
3227
|
+
async function runtimeFingerprint() {
|
|
3228
|
+
const bundledProject2 = resolve11(packageRoot2, "training/local-runner");
|
|
3229
|
+
return hashJson({
|
|
3230
|
+
tt_local_version: await packageVersion2(),
|
|
3231
|
+
node_version: process.version,
|
|
3232
|
+
platform: process.platform,
|
|
3233
|
+
architecture: process.arch,
|
|
3234
|
+
runner_pyproject: await hashFileIfPresent(resolve11(bundledProject2, "pyproject.toml")),
|
|
3235
|
+
runner_uv_lock: await hashFileIfPresent(resolve11(bundledProject2, "uv.lock"))
|
|
3236
|
+
});
|
|
3237
|
+
}
|
|
3238
|
+
function preparedSourceFingerprint(args) {
|
|
3239
|
+
return hashJson({
|
|
3240
|
+
request_fingerprint: args.requestFingerprint,
|
|
3241
|
+
runtime_fingerprint: args.runtimeFingerprint,
|
|
3242
|
+
preparation_config: args.preparationConfig,
|
|
3243
|
+
base_model_revision: args.baseModelRevision ?? null,
|
|
3244
|
+
base_model_fingerprint: args.baseModelFingerprint ?? null,
|
|
3245
|
+
dataset_fingerprints: args.datasetFingerprints
|
|
3246
|
+
});
|
|
3247
|
+
}
|
|
3248
|
+
async function hashFile(path) {
|
|
3249
|
+
const hash = createHash5("sha256");
|
|
3250
|
+
for await (const chunk of createReadStream2(path)) hash.update(chunk);
|
|
3251
|
+
return hash.digest("hex");
|
|
3252
|
+
}
|
|
3253
|
+
async function datasetFingerprints(request) {
|
|
3254
|
+
const dataset = request.dataset_prebuilt;
|
|
3255
|
+
const entries = [
|
|
3256
|
+
["training", dataset?.training],
|
|
3257
|
+
["validation", dataset?.validation],
|
|
3258
|
+
["test", dataset?.test]
|
|
3259
|
+
];
|
|
3260
|
+
const fingerprints = {};
|
|
3261
|
+
for (const [key, value] of entries) {
|
|
3262
|
+
if (!value) continue;
|
|
3263
|
+
const datasetPath = stripFileUri(value);
|
|
3264
|
+
fingerprints[key] = await hashFile(datasetPath);
|
|
3265
|
+
}
|
|
3266
|
+
return fingerprints;
|
|
3267
|
+
}
|
|
3268
|
+
async function resolveBaseModelRevision(request, config) {
|
|
3269
|
+
const explicit = request.hyperparameters.base_model_revision;
|
|
3270
|
+
if (explicit) return explicit;
|
|
3271
|
+
if (config.paths.baseModel) {
|
|
3272
|
+
const match = resolve11(config.paths.baseModel).match(/[\\/]snapshots[\\/]([^\\/]+)(?:[\\/]|$)/);
|
|
3273
|
+
if (match?.[1]) return match[1];
|
|
3274
|
+
}
|
|
3275
|
+
if (!request.spec_snapshot.base_model.includes("/")) return void 0;
|
|
3276
|
+
const repository = `models--${request.spec_snapshot.base_model.replaceAll("/", "--")}`;
|
|
3277
|
+
const cacheEnvironment = withHuggingFaceCacheEnvironment(process.env, config.paths.modelCache);
|
|
3278
|
+
const refPath = resolve11(cacheEnvironment.HF_HUB_CACHE, repository, "refs", "main");
|
|
3279
|
+
try {
|
|
3280
|
+
const revision = (await readFile8(refPath, "utf8")).trim();
|
|
3281
|
+
return revision || void 0;
|
|
3282
|
+
} catch {
|
|
3283
|
+
return void 0;
|
|
3284
|
+
}
|
|
3285
|
+
}
|
|
3286
|
+
async function fingerprintLocalBaseModel(uri) {
|
|
3287
|
+
const root = localModelArtifactPath(uri);
|
|
3288
|
+
const rootMetadata = await lstat4(root);
|
|
3289
|
+
if (rootMetadata.isSymbolicLink()) {
|
|
3290
|
+
throw new Error(`Local base model path must not itself be a symbolic link: ${root}`);
|
|
3291
|
+
}
|
|
3292
|
+
if (!rootMetadata.isDirectory()) {
|
|
3293
|
+
throw new Error(`Local base model must be a Hugging Face snapshot directory: ${root}`);
|
|
3294
|
+
}
|
|
3295
|
+
await verifyLocalBaseModel(root);
|
|
3296
|
+
const files = [];
|
|
3297
|
+
const visit = async (directory) => {
|
|
3298
|
+
for (const entry of await readdir5(directory, { withFileTypes: true })) {
|
|
3299
|
+
const child = resolve11(directory, entry.name);
|
|
3300
|
+
if (entry.isDirectory()) {
|
|
3301
|
+
await visit(child);
|
|
3302
|
+
continue;
|
|
3303
|
+
}
|
|
3304
|
+
const childMetadata = await stat5(child);
|
|
3305
|
+
if (entry.isSymbolicLink() && !childMetadata.isFile()) {
|
|
3306
|
+
throw new Error(`Local base model contains a non-file symbolic link: ${child}`);
|
|
3307
|
+
}
|
|
3308
|
+
if (!childMetadata.isFile()) continue;
|
|
3309
|
+
files.push({
|
|
3310
|
+
path: relative2(root, child).split("\\").join("/"),
|
|
3311
|
+
size_bytes: childMetadata.size,
|
|
3312
|
+
sha256: await hashFile(child)
|
|
3313
|
+
});
|
|
3314
|
+
}
|
|
3315
|
+
};
|
|
3316
|
+
await visit(root);
|
|
3317
|
+
files.sort((left, right) => left.path.localeCompare(right.path));
|
|
3318
|
+
if (files.length === 0) throw new Error(`Local base model directory is empty: ${root}`);
|
|
3319
|
+
return hashJson({ kind: "directory", files });
|
|
3320
|
+
}
|
|
3321
|
+
function statusForProgressStage(stage) {
|
|
3322
|
+
if (stage === "evaluating_baseline") return "evaluating_baseline";
|
|
3323
|
+
if (stage === "evaluating_candidate") return "evaluating_candidate";
|
|
3324
|
+
if (stage === "training") return "training";
|
|
3325
|
+
if (stage === "preparing") return "preparing";
|
|
3326
|
+
if (stage === "reporting") return "reporting";
|
|
3327
|
+
return "training";
|
|
3328
|
+
}
|
|
3329
|
+
async function readStageMetadata(path) {
|
|
3330
|
+
try {
|
|
3331
|
+
const metadata = await readJson(path);
|
|
3332
|
+
return typeof metadata.source_fingerprint === "string" ? metadata : null;
|
|
3333
|
+
} catch {
|
|
3334
|
+
return null;
|
|
3335
|
+
}
|
|
3336
|
+
}
|
|
3337
|
+
async function removePrefixedArtifacts(path) {
|
|
3338
|
+
const directory = dirname8(path);
|
|
3339
|
+
const prefix = `${basename(path)}.`;
|
|
3340
|
+
const names = await readdir5(directory).catch(() => []);
|
|
3341
|
+
await Promise.all([
|
|
3342
|
+
rm3(path, { force: true }),
|
|
3343
|
+
...names.filter((name) => name.startsWith(prefix)).map((name) => rm3(resolve11(directory, name), { recursive: true, force: true }))
|
|
3344
|
+
]);
|
|
3345
|
+
}
|
|
3346
|
+
async function cleanupStageArtifacts(artifacts, stage) {
|
|
3347
|
+
const removeReport = async () => rm3(artifacts.runReportJson, { force: true });
|
|
3348
|
+
if (stage === "prepare") {
|
|
3349
|
+
await Promise.all([
|
|
3350
|
+
removePrefixedArtifacts(artifacts.baselineEvalJson),
|
|
3351
|
+
removePrefixedArtifacts(artifacts.candidateEvalJson),
|
|
3352
|
+
removePrefixedArtifacts(artifacts.generalBaselineEvalJson),
|
|
3353
|
+
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
3354
|
+
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
3355
|
+
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
3356
|
+
rm3(resolve11(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
3357
|
+
removeReport(),
|
|
3358
|
+
rm3(artifacts.artifactManifestJson, { force: true })
|
|
3359
|
+
]);
|
|
3360
|
+
return;
|
|
3361
|
+
}
|
|
3362
|
+
if (stage === "baseline") {
|
|
3363
|
+
await Promise.all([
|
|
3364
|
+
removePrefixedArtifacts(artifacts.baselineEvalJson),
|
|
3365
|
+
removePrefixedArtifacts(artifacts.generalBaselineEvalJson),
|
|
3366
|
+
removeReport()
|
|
3367
|
+
]);
|
|
3368
|
+
return;
|
|
3369
|
+
}
|
|
3370
|
+
if (stage === "train") {
|
|
3371
|
+
await Promise.all([
|
|
3372
|
+
rm3(artifacts.trainingDir, { recursive: true, force: true }),
|
|
3373
|
+
removePrefixedArtifacts(artifacts.trainingReportJson),
|
|
3374
|
+
rm3(resolve11(artifacts.runDir, "model.tar.gz"), { force: true }),
|
|
3375
|
+
removePrefixedArtifacts(artifacts.candidateEvalJson),
|
|
3376
|
+
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
3377
|
+
removeReport()
|
|
3378
|
+
]);
|
|
3379
|
+
await prepareRunDirectories(artifacts);
|
|
3380
|
+
return;
|
|
3381
|
+
}
|
|
3382
|
+
if (stage === "candidate") {
|
|
3383
|
+
await Promise.all([
|
|
3384
|
+
removePrefixedArtifacts(artifacts.candidateEvalJson),
|
|
3385
|
+
removePrefixedArtifacts(artifacts.generalCandidateEvalJson),
|
|
3386
|
+
removeReport()
|
|
3387
|
+
]);
|
|
3388
|
+
return;
|
|
3389
|
+
}
|
|
3390
|
+
await removeReport();
|
|
3391
|
+
}
|
|
3392
|
+
function manifestRelativePath(artifacts, path) {
|
|
3393
|
+
return relative2(artifacts.runDir, path).split("\\").join("/");
|
|
3394
|
+
}
|
|
3395
|
+
async function verifyReusableArtifacts(artifacts, paths, options = {}) {
|
|
3396
|
+
if (!await pathExists2(artifacts.artifactManifestJson)) return false;
|
|
3397
|
+
await assertArtifactManifest(artifacts.artifactManifestJson, {
|
|
3398
|
+
requiredPaths: paths.map((path) => manifestRelativePath(artifacts, path)),
|
|
3399
|
+
scopeToRequired: true,
|
|
3400
|
+
verifyModel: options.verifyModel
|
|
3401
|
+
});
|
|
3402
|
+
return true;
|
|
3403
|
+
}
|
|
3404
|
+
async function throwIfCancelled(store, request) {
|
|
3405
|
+
if (await store.isCancellationRequested(request.run_id)) {
|
|
3406
|
+
throw new ProcessCancelledError(`Run ${request.run_id} was cancelled.`);
|
|
3407
|
+
}
|
|
3408
|
+
}
|
|
3409
|
+
async function acquireWorkflowLock(lockPath, description) {
|
|
3410
|
+
await mkdir8(dirname8(lockPath), { recursive: true });
|
|
3411
|
+
const token = randomUUID4();
|
|
3412
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
3413
|
+
try {
|
|
3414
|
+
const handle = await open2(lockPath, "wx", 384);
|
|
3415
|
+
try {
|
|
3416
|
+
await handle.writeFile(`${JSON.stringify({ token, pid: process.pid, created_at: (/* @__PURE__ */ new Date()).toISOString() })}
|
|
3417
|
+
`);
|
|
3418
|
+
} finally {
|
|
3419
|
+
await handle.close();
|
|
3420
|
+
}
|
|
3421
|
+
return async () => {
|
|
3422
|
+
try {
|
|
3423
|
+
const owner = JSON.parse(await readFile8(lockPath, "utf8"));
|
|
3424
|
+
if (owner.token === token) await rm3(lockPath, { force: true });
|
|
3425
|
+
} catch {
|
|
3426
|
+
}
|
|
3427
|
+
};
|
|
3428
|
+
} catch (error) {
|
|
3429
|
+
if (error.code !== "EEXIST") throw error;
|
|
3430
|
+
const metadata = await stat5(lockPath).catch(() => null);
|
|
3431
|
+
let owner = null;
|
|
3432
|
+
try {
|
|
3433
|
+
owner = JSON.parse(await readFile8(lockPath, "utf8"));
|
|
3434
|
+
} catch {
|
|
3435
|
+
owner = null;
|
|
3436
|
+
}
|
|
3437
|
+
let ownerAlive = false;
|
|
3438
|
+
if (typeof owner?.pid === "number" && Number.isSafeInteger(owner.pid) && owner.pid > 0) {
|
|
3439
|
+
try {
|
|
3440
|
+
process.kill(owner.pid, 0);
|
|
3441
|
+
ownerAlive = true;
|
|
3442
|
+
} catch (killError) {
|
|
3443
|
+
ownerAlive = killError.code !== "ESRCH";
|
|
3444
|
+
}
|
|
3445
|
+
}
|
|
3446
|
+
const recent = metadata ? Date.now() - metadata.mtimeMs < 1e4 : true;
|
|
3447
|
+
if (ownerAlive || !owner && recent) {
|
|
3448
|
+
throw new Error(
|
|
3449
|
+
`${description} already has an active local workflow (lock: ${lockPath}).`
|
|
3450
|
+
);
|
|
3451
|
+
}
|
|
3452
|
+
const stalePath = `${lockPath}.stale.${randomUUID4()}`;
|
|
3453
|
+
try {
|
|
3454
|
+
await rename3(lockPath, stalePath);
|
|
3455
|
+
await rm3(stalePath, { force: true });
|
|
3456
|
+
} catch (renameError) {
|
|
3457
|
+
if (renameError.code !== "ENOENT") throw renameError;
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
}
|
|
3461
|
+
throw new Error(`Unable to acquire local workflow lock for ${description}.`);
|
|
3462
|
+
}
|
|
3463
|
+
async function acquireRunLock(store, runId) {
|
|
3464
|
+
return acquireWorkflowLock(resolve11(store.paths.runsDir, runId, "workflow.lock"), `Run ${runId}`);
|
|
3465
|
+
}
|
|
3466
|
+
async function acquireArtifactLock(artifacts) {
|
|
3467
|
+
return acquireWorkflowLock(
|
|
3468
|
+
join8(artifacts.runDir, ARTIFACT_WORKFLOW_LOCK_FILE),
|
|
3469
|
+
`Artifact directory ${artifacts.runDir}`
|
|
3470
|
+
);
|
|
3471
|
+
}
|
|
3472
|
+
function isDryTraining(training) {
|
|
3473
|
+
return training.metrics?.dry_run === true;
|
|
3474
|
+
}
|
|
3475
|
+
async function modelManifestContract(prepared, training) {
|
|
3476
|
+
if (isDryTraining(training) || !training.model_artifact_uri) return void 0;
|
|
3477
|
+
const inspection = await assertUsableModelArtifact(training.model_artifact_uri);
|
|
3478
|
+
const adapterWeights = inspection.adapter_weight_file_count > 0 && inspection.adapter_weight_bytes > 0;
|
|
3479
|
+
if (!adapterWeights || !inspection.has_adapter_config) {
|
|
3480
|
+
throw new Error(
|
|
3481
|
+
`Local training must produce a PEFT adapter with adapter_model.safetensors or adapter_model.bin and a non-empty adapter_config.json: ${inspection.path}`
|
|
3482
|
+
);
|
|
3483
|
+
}
|
|
3484
|
+
const format = inspection.kind === "file" ? "tar.gz" : "huggingface-directory";
|
|
3485
|
+
return {
|
|
3486
|
+
artifact_kind: inspection.kind,
|
|
3487
|
+
format,
|
|
3488
|
+
framework: "transformers-peft",
|
|
3489
|
+
base_model: prepared.request.spec_snapshot.base_model,
|
|
3490
|
+
base_model_revision: prepared.metadata.base_model_revision ?? prepared.request.hyperparameters.base_model_revision,
|
|
3491
|
+
base_model_artifact_uri: training.base_model_artifact_uri,
|
|
3492
|
+
base_model_fingerprint: prepared.metadata.base_model_fingerprint ?? void 0,
|
|
3493
|
+
artifact_uri: training.model_artifact_uri,
|
|
3494
|
+
artifact_root: inspection.path,
|
|
3495
|
+
servable: true
|
|
3496
|
+
};
|
|
3497
|
+
}
|
|
3498
|
+
async function refreshArtifactManifest(prepared) {
|
|
3499
|
+
let model;
|
|
3500
|
+
if (await pathExists2(prepared.artifacts.trainingReportJson)) {
|
|
3501
|
+
const training = trainingReportSchema.parse(await readJson(prepared.artifacts.trainingReportJson));
|
|
3502
|
+
model = await modelManifestContract(prepared, training);
|
|
3503
|
+
}
|
|
3504
|
+
await writeArtifactManifest(prepared.artifacts, { model });
|
|
3505
|
+
}
|
|
3506
|
+
function stageOutputPath(prepared, stage) {
|
|
3507
|
+
if (stage === "baseline") return prepared.artifacts.baselineEvalJson;
|
|
3508
|
+
if (stage === "train") return prepared.artifacts.trainingReportJson;
|
|
3509
|
+
return prepared.artifacts.candidateEvalJson;
|
|
3510
|
+
}
|
|
3511
|
+
function stageFingerprintPath(prepared, stage) {
|
|
3512
|
+
return `${stageOutputPath(prepared, stage)}.stage.json`;
|
|
3513
|
+
}
|
|
3514
|
+
async function runnerFileFingerprints() {
|
|
3515
|
+
const project = resolve11(packageRoot2, "training/local-runner");
|
|
3516
|
+
const bundledSource = resolve11(packageRoot2, "training/local-runner/src");
|
|
3517
|
+
const pythonSources = (await readdir5(bundledSource, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".py")).map((entry) => resolve11(bundledSource, entry.name));
|
|
3518
|
+
const candidates = [.../* @__PURE__ */ new Set([
|
|
3519
|
+
...pythonSources,
|
|
3520
|
+
resolve11(project, "pyproject.toml"),
|
|
3521
|
+
resolve11(project, "uv.lock")
|
|
3522
|
+
])];
|
|
3523
|
+
const fingerprints = {};
|
|
3524
|
+
for (const path of candidates.sort()) fingerprints[path] = await hashFileIfPresent(path);
|
|
3525
|
+
return fingerprints;
|
|
3526
|
+
}
|
|
3527
|
+
async function stageFingerprint(args) {
|
|
3528
|
+
const common = {
|
|
3529
|
+
source_fingerprint: args.prepared.metadata.source_fingerprint,
|
|
3530
|
+
runtime_fingerprint: args.prepared.metadata.runtime_fingerprint,
|
|
3531
|
+
dry_run: args.config.dryRun
|
|
3532
|
+
};
|
|
3533
|
+
if (args.stage === "train") {
|
|
3534
|
+
const entrypointFiles2 = await runnerFileFingerprints();
|
|
3535
|
+
return hashJson({
|
|
3536
|
+
...common,
|
|
3537
|
+
paths: args.config.paths,
|
|
3538
|
+
entrypoint_files: entrypointFiles2
|
|
3539
|
+
});
|
|
3540
|
+
}
|
|
3541
|
+
const entrypointFiles = await runnerFileFingerprints();
|
|
3542
|
+
const evaluation = {
|
|
3543
|
+
inference_protocol_version: INFERENCE_PROTOCOL_VERSION,
|
|
3544
|
+
inference: args.config.evaluation.inference,
|
|
3545
|
+
scoring: args.config.evaluation.scoring,
|
|
3546
|
+
timeout_ms: args.config.evaluation.timeoutMs,
|
|
3547
|
+
baseline_cache: args.config.evaluation.baselineCache,
|
|
3548
|
+
general_regression: args.prepared.generalRegression ? {
|
|
3549
|
+
dataset_sha256: args.prepared.generalRegression.datasetSha256,
|
|
3550
|
+
system_prompt: args.prepared.generalRegression.system
|
|
3551
|
+
} : null,
|
|
3552
|
+
model_cache: args.config.paths.modelCache
|
|
3553
|
+
};
|
|
3554
|
+
return hashJson({
|
|
3555
|
+
...common,
|
|
3556
|
+
evaluation,
|
|
3557
|
+
entrypoint_files: entrypointFiles
|
|
3558
|
+
});
|
|
3559
|
+
}
|
|
3560
|
+
async function writeStageFingerprint(args) {
|
|
3561
|
+
await writeJsonAtomic(stageFingerprintPath(args.prepared, args.stage), {
|
|
3562
|
+
schema_version: 1,
|
|
3563
|
+
stage: args.stage,
|
|
3564
|
+
fingerprint: await stageFingerprint(args),
|
|
3565
|
+
written_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
3566
|
+
});
|
|
3567
|
+
}
|
|
3568
|
+
async function hasCurrentStageFingerprint(args) {
|
|
3569
|
+
try {
|
|
3570
|
+
const record = await readJson(stageFingerprintPath(args.prepared, args.stage));
|
|
3571
|
+
return record.fingerprint === await stageFingerprint(args);
|
|
3572
|
+
} catch {
|
|
3573
|
+
return false;
|
|
3574
|
+
}
|
|
3575
|
+
}
|
|
3576
|
+
async function canReuseStageArtifact(args) {
|
|
3577
|
+
const output = stageOutputPath(args.prepared, args.stage);
|
|
3578
|
+
if (!await pathExists2(output) || !await hasCurrentStageFingerprint(args)) return false;
|
|
3579
|
+
return verifyReusableArtifacts(
|
|
3580
|
+
args.prepared.artifacts,
|
|
3581
|
+
[output, stageFingerprintPath(args.prepared, args.stage), ...args.additionalPaths ?? []],
|
|
3582
|
+
{ verifyModel: args.verifyModel }
|
|
3583
|
+
);
|
|
3584
|
+
}
|
|
3585
|
+
function createStoreReporter(input) {
|
|
3586
|
+
return {
|
|
3587
|
+
verbose: input.reporter?.verbose,
|
|
3588
|
+
async onEvent(event) {
|
|
3589
|
+
const state = await input.store.updateRun({
|
|
3590
|
+
runId: input.request.run_id,
|
|
3591
|
+
status: statusForProgressStage(event.stage),
|
|
3592
|
+
stage: event.stage,
|
|
3593
|
+
message: event.message,
|
|
3594
|
+
details: event.details
|
|
3595
|
+
});
|
|
3596
|
+
if (state.status === "cancelled") throw new ProcessCancelledError(`Run ${input.request.run_id} was cancelled.`);
|
|
3597
|
+
await input.reporter?.onEvent?.(event);
|
|
3598
|
+
},
|
|
3599
|
+
async onLog(log) {
|
|
3600
|
+
await input.reporter?.onLog?.(log);
|
|
3601
|
+
}
|
|
3602
|
+
};
|
|
3603
|
+
}
|
|
3604
|
+
async function updateRun(input) {
|
|
3605
|
+
const state = await input.store.updateRun({
|
|
3606
|
+
runId: input.request.run_id,
|
|
3607
|
+
status: input.status,
|
|
3608
|
+
stage: input.stage,
|
|
3609
|
+
message: input.message,
|
|
3610
|
+
details: input.details
|
|
3611
|
+
});
|
|
3612
|
+
if (state.status === "cancelled") throw new ProcessCancelledError(`Run ${input.request.run_id} was cancelled.`);
|
|
3613
|
+
await input.reporter?.onEvent?.({
|
|
3614
|
+
stage: input.stage,
|
|
3615
|
+
status: input.status,
|
|
3616
|
+
message: input.message,
|
|
3617
|
+
details: input.details
|
|
3618
|
+
});
|
|
3619
|
+
return state;
|
|
3620
|
+
}
|
|
3621
|
+
async function ensureRunRecord(args) {
|
|
3622
|
+
await prepareRunDirectories(args.artifacts);
|
|
3623
|
+
let existing = null;
|
|
3624
|
+
try {
|
|
3625
|
+
existing = await args.store.getRun(args.request.run_id);
|
|
3626
|
+
} catch (error) {
|
|
3627
|
+
if (!(error instanceof Error) || !error.message.startsWith("Run not found:")) throw error;
|
|
3628
|
+
}
|
|
3629
|
+
if (existing) {
|
|
3630
|
+
if (existing.behavior_spec_id !== args.request.behavior_spec_id || existing.user_id !== args.request.user_id || existing.run_number !== args.request.run_number) {
|
|
3631
|
+
throw new Error(
|
|
3632
|
+
`Run ${args.request.run_id} cannot be reused with different user, behavior spec, or run number identity.`
|
|
3633
|
+
);
|
|
3634
|
+
}
|
|
3635
|
+
if (resolve11(existing.artifact_dir) !== resolve11(args.artifacts.runDir)) {
|
|
3636
|
+
throw new Error(
|
|
3637
|
+
`Run ${args.request.run_id} is already bound to artifact directory ${existing.artifact_dir}; it cannot be resumed with ${args.artifacts.runDir}.`
|
|
3638
|
+
);
|
|
3639
|
+
}
|
|
3640
|
+
} else {
|
|
3641
|
+
await args.store.startRun({ request: args.request, artifactDir: args.artifacts.runDir });
|
|
3642
|
+
await args.reporter?.onEvent?.({
|
|
3643
|
+
stage: "queued",
|
|
3644
|
+
status: "queued",
|
|
3645
|
+
message: "Run queued.",
|
|
3646
|
+
details: { run_id: args.request.run_id, artifact_dir: args.artifacts.runDir }
|
|
3647
|
+
});
|
|
3648
|
+
}
|
|
3649
|
+
}
|
|
3650
|
+
async function computePreparedRun(args) {
|
|
3651
|
+
const { request, config, artifacts } = args;
|
|
3652
|
+
const evalSampleSeed = config.evaluation.sampleSeed ?? deriveSampleSeed(request.behavior_spec_id);
|
|
3653
|
+
const dataset = await validateDatasetInputs(request, config);
|
|
3654
|
+
const examples = dataset.evaluationExamples;
|
|
3655
|
+
if (args.writeArtifacts) {
|
|
3656
|
+
await writeFileAtomic(artifacts.trainingJsonl, `${dataset.trainingJsonl}
|
|
3657
|
+
`);
|
|
3658
|
+
}
|
|
3659
|
+
const system = buildSystemMessage(request.spec_snapshot);
|
|
3660
|
+
const baseModelForEvaluation = config.paths.baseModel ?? request.spec_snapshot.base_model;
|
|
3661
|
+
const maxEvalExamples = config.evaluation.maxExamples ?? request.hyperparameters.max_eval_examples;
|
|
3662
|
+
const evalExamplesUsed = Math.min(maxEvalExamples ?? examples.length, examples.length);
|
|
3663
|
+
const fingerprints = await datasetFingerprints(request);
|
|
3664
|
+
const generalRegressionConfig = config.evaluation.generalRegression;
|
|
3665
|
+
const generalRegression = generalRegressionConfig ? {
|
|
3666
|
+
datasetPath: generalRegressionConfig.dataset,
|
|
3667
|
+
datasetSha256: await hashFile(generalRegressionConfig.dataset),
|
|
3668
|
+
...await evaluationSuiteFromChatJsonl(
|
|
3669
|
+
generalRegressionConfig.dataset,
|
|
3670
|
+
generalRegressionConfig.systemPrompt
|
|
3671
|
+
)
|
|
3672
|
+
} : void 0;
|
|
3673
|
+
const requestFingerprint = hashJson(request);
|
|
3674
|
+
const runtimeFingerprintValue = await runtimeFingerprint();
|
|
3675
|
+
const baseModelRevision = await resolveBaseModelRevision(request, config);
|
|
3676
|
+
const baseModelFingerprint = config.paths.baseModel ? await fingerprintLocalBaseModel(config.paths.baseModel) : void 0;
|
|
3677
|
+
const metadata = {
|
|
3678
|
+
run_id: request.run_id,
|
|
3679
|
+
behavior_spec_id: request.behavior_spec_id,
|
|
3680
|
+
user_id: request.user_id,
|
|
3681
|
+
request_fingerprint: requestFingerprint,
|
|
3682
|
+
runtime_fingerprint: runtimeFingerprintValue,
|
|
3683
|
+
source_fingerprint: preparedSourceFingerprint({
|
|
3684
|
+
requestFingerprint,
|
|
3685
|
+
runtimeFingerprint: runtimeFingerprintValue,
|
|
3686
|
+
preparationConfig: {
|
|
3687
|
+
dry_run: config.dryRun,
|
|
3688
|
+
base_model_path: config.paths.baseModel,
|
|
3689
|
+
max_eval_examples: config.evaluation.maxExamples,
|
|
3690
|
+
eval_sample_seed: config.evaluation.sampleSeed
|
|
3691
|
+
},
|
|
3692
|
+
baseModelRevision,
|
|
3693
|
+
baseModelFingerprint,
|
|
3694
|
+
datasetFingerprints: fingerprints
|
|
3695
|
+
}),
|
|
3696
|
+
eval_split: dataset.evalSplit,
|
|
3697
|
+
eval_sample_seed: evalSampleSeed,
|
|
3698
|
+
eval_examples_total: examples.length,
|
|
3699
|
+
eval_examples_used: evalExamplesUsed,
|
|
3700
|
+
max_eval_examples: maxEvalExamples ?? null,
|
|
3701
|
+
training_example_count: dataset.trainingExampleCount,
|
|
3702
|
+
dataset_prebuilt: Boolean(request.dataset_prebuilt),
|
|
3703
|
+
dataset_format: request.dataset_prebuilt?.format ?? null,
|
|
3704
|
+
dataset_fingerprints: fingerprints,
|
|
3705
|
+
dataset_uri: fileUri(artifacts.trainingJsonl),
|
|
3706
|
+
base_model_for_evaluation: baseModelForEvaluation,
|
|
3707
|
+
base_model_revision: baseModelRevision ?? null,
|
|
3708
|
+
base_model_fingerprint: baseModelFingerprint ?? null,
|
|
3709
|
+
system_prompt_sha256: createHash5("sha256").update(system).digest("hex"),
|
|
3710
|
+
prepared_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
3711
|
+
};
|
|
3712
|
+
if (args.writeArtifacts) await writeJsonAtomic(artifacts.stageMetadataJson, metadata);
|
|
3713
|
+
return {
|
|
3714
|
+
request,
|
|
3715
|
+
artifacts,
|
|
3716
|
+
metadata,
|
|
3717
|
+
examples,
|
|
3718
|
+
system,
|
|
3719
|
+
baseModelForEvaluation,
|
|
3720
|
+
maxEvalExamples,
|
|
3721
|
+
generalRegression
|
|
3722
|
+
};
|
|
3723
|
+
}
|
|
3724
|
+
function generalRegressionEvaluationConfig(config) {
|
|
3725
|
+
const { maxExamples: _maxExamples, sampleSeed: _sampleSeed, ...evaluation } = config.evaluation;
|
|
3726
|
+
return { ...config, evaluation };
|
|
3727
|
+
}
|
|
3728
|
+
function generalRegressionRequiredPaths(prepared, kind) {
|
|
3729
|
+
if (!prepared.generalRegression) return [];
|
|
3730
|
+
return [
|
|
3731
|
+
kind === "baseline" ? prepared.artifacts.generalBaselineEvalJson : prepared.artifacts.generalCandidateEvalJson
|
|
3732
|
+
];
|
|
3733
|
+
}
|
|
3734
|
+
async function prepareStage(args) {
|
|
3735
|
+
const preparedExists = await pathExists2(args.artifacts.stageMetadataJson) && await pathExists2(args.artifacts.trainingJsonl);
|
|
3736
|
+
const prepared = await computePreparedRun({ ...args, writeArtifacts: false });
|
|
3737
|
+
await args.store.syncRunRequest(args.request, args.artifacts.runDir);
|
|
3738
|
+
await writeJsonAtomic(resolve11(args.artifacts.runDir, "request.json"), args.request);
|
|
3739
|
+
const existingMetadata = preparedExists ? await readStageMetadata(args.artifacts.stageMetadataJson) : null;
|
|
3740
|
+
let canReuse = preparedExists && existingMetadata?.source_fingerprint === prepared.metadata.source_fingerprint;
|
|
3741
|
+
if (canReuse) {
|
|
3742
|
+
canReuse = await verifyReusableArtifacts(args.artifacts, [
|
|
3743
|
+
args.artifacts.stageMetadataJson,
|
|
3744
|
+
args.artifacts.trainingJsonl
|
|
3745
|
+
]);
|
|
3746
|
+
}
|
|
3747
|
+
if (canReuse) {
|
|
3748
|
+
await throwIfCancelled(args.store, args.request);
|
|
3749
|
+
await updateRun({
|
|
3750
|
+
...args,
|
|
3751
|
+
status: "preparing",
|
|
3752
|
+
stage: "preparing",
|
|
3753
|
+
message: "Reusing prepared local run artifacts.",
|
|
3754
|
+
details: { artifact_dir: args.artifacts.runDir }
|
|
3755
|
+
});
|
|
3756
|
+
return prepared;
|
|
3757
|
+
}
|
|
3758
|
+
await updateRun({
|
|
3759
|
+
...args,
|
|
3760
|
+
status: "preparing",
|
|
3761
|
+
stage: "preparing",
|
|
3762
|
+
message: "Preparing local run artifacts.",
|
|
3763
|
+
details: { artifact_dir: args.artifacts.runDir }
|
|
3764
|
+
});
|
|
3765
|
+
await throwIfCancelled(args.store, args.request);
|
|
3766
|
+
await args.store.invalidateRunOutputs(args.request.run_id, { report: true, model: true });
|
|
3767
|
+
await cleanupStageArtifacts(args.artifacts, "prepare");
|
|
3768
|
+
const refreshed = await computePreparedRun({ ...args, writeArtifacts: true });
|
|
3769
|
+
await refreshArtifactManifest(refreshed);
|
|
3770
|
+
return refreshed;
|
|
3771
|
+
}
|
|
3772
|
+
async function runBaselineStage(args) {
|
|
3773
|
+
if (await canReuseStageArtifact({
|
|
3774
|
+
stage: "baseline",
|
|
3775
|
+
prepared: args.prepared,
|
|
3776
|
+
config: args.config,
|
|
3777
|
+
additionalPaths: generalRegressionRequiredPaths(args.prepared, "baseline")
|
|
3778
|
+
})) {
|
|
3779
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3780
|
+
await updateRun({
|
|
3781
|
+
store: args.store,
|
|
3782
|
+
reporter: args.reporter,
|
|
3783
|
+
request: args.prepared.request,
|
|
3784
|
+
status: "evaluating_baseline",
|
|
3785
|
+
stage: "evaluating_baseline",
|
|
3786
|
+
message: "Reusing existing baseline evaluation.",
|
|
3787
|
+
details: { path: args.prepared.artifacts.baselineEvalJson }
|
|
3788
|
+
});
|
|
3789
|
+
const report2 = evalReportSchema.parse(await readJson(args.prepared.artifacts.baselineEvalJson));
|
|
3790
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3791
|
+
return report2;
|
|
3792
|
+
}
|
|
3793
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3794
|
+
await args.store.invalidateRunOutputs(args.prepared.request.run_id, { report: true });
|
|
3795
|
+
await cleanupStageArtifacts(args.prepared.artifacts, "baseline");
|
|
3796
|
+
await updateRun({
|
|
3797
|
+
store: args.store,
|
|
3798
|
+
reporter: args.reporter,
|
|
3799
|
+
request: args.prepared.request,
|
|
3800
|
+
status: "evaluating_baseline",
|
|
3801
|
+
stage: "evaluating_baseline",
|
|
3802
|
+
message: "Running baseline evaluation.",
|
|
3803
|
+
details: {
|
|
3804
|
+
examples: args.prepared.examples.length,
|
|
3805
|
+
eval_examples_used: args.prepared.metadata.eval_examples_used,
|
|
3806
|
+
eval_split: args.prepared.metadata.eval_split,
|
|
3807
|
+
model_id: args.prepared.baseModelForEvaluation
|
|
3808
|
+
}
|
|
3809
|
+
});
|
|
3810
|
+
const report = await evaluateExamples({
|
|
3811
|
+
kind: "baseline",
|
|
3812
|
+
modelId: args.prepared.baseModelForEvaluation,
|
|
3813
|
+
baseModelId: args.prepared.baseModelForEvaluation,
|
|
3814
|
+
baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
|
|
3815
|
+
sourceFingerprint: args.prepared.metadata.base_model_fingerprint ?? void 0,
|
|
3816
|
+
examples: args.prepared.examples,
|
|
3817
|
+
system: args.prepared.system,
|
|
3818
|
+
config: args.config,
|
|
3819
|
+
outputPath: args.prepared.artifacts.baselineEvalJson,
|
|
3820
|
+
reporter: args.runReporter,
|
|
3821
|
+
maxExamples: args.prepared.maxEvalExamples,
|
|
3822
|
+
evalSplit: args.prepared.metadata.eval_split,
|
|
3823
|
+
sampleSeed: args.prepared.metadata.eval_sample_seed,
|
|
3824
|
+
shouldCancel: () => args.store.isCancellationRequested(args.prepared.request.run_id)
|
|
3825
|
+
});
|
|
3826
|
+
if (args.prepared.generalRegression) {
|
|
3827
|
+
await updateRun({
|
|
3828
|
+
store: args.store,
|
|
3829
|
+
reporter: args.reporter,
|
|
3830
|
+
request: args.prepared.request,
|
|
3831
|
+
status: "evaluating_baseline",
|
|
3832
|
+
stage: "evaluating_baseline",
|
|
3833
|
+
message: "Running baseline general regression evaluation.",
|
|
3834
|
+
details: {
|
|
3835
|
+
examples: args.prepared.generalRegression.examples.length,
|
|
3836
|
+
dataset: args.prepared.generalRegression.datasetPath
|
|
3837
|
+
}
|
|
3838
|
+
});
|
|
3839
|
+
await evaluateExamples({
|
|
3840
|
+
kind: "baseline",
|
|
3841
|
+
modelId: args.prepared.baseModelForEvaluation,
|
|
3842
|
+
baseModelId: args.prepared.baseModelForEvaluation,
|
|
3843
|
+
baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
|
|
3844
|
+
sourceFingerprint: args.prepared.metadata.base_model_fingerprint ?? void 0,
|
|
3845
|
+
examples: args.prepared.generalRegression.examples,
|
|
3846
|
+
system: args.prepared.generalRegression.system,
|
|
3847
|
+
config: generalRegressionEvaluationConfig(args.config),
|
|
3848
|
+
outputPath: args.prepared.artifacts.generalBaselineEvalJson,
|
|
3849
|
+
reporter: args.runReporter,
|
|
3850
|
+
evalSplit: "general_regression",
|
|
3851
|
+
shouldCancel: () => args.store.isCancellationRequested(args.prepared.request.run_id)
|
|
3852
|
+
});
|
|
3853
|
+
}
|
|
3854
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3855
|
+
await writeStageFingerprint({ stage: "baseline", prepared: args.prepared, config: args.config });
|
|
3856
|
+
return report;
|
|
3857
|
+
}
|
|
3858
|
+
async function runTrainStage(args) {
|
|
3859
|
+
if (await canReuseStageArtifact({
|
|
3860
|
+
stage: "train",
|
|
3861
|
+
prepared: args.prepared,
|
|
3862
|
+
config: args.config,
|
|
3863
|
+
verifyModel: true
|
|
3864
|
+
})) {
|
|
3865
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3866
|
+
await updateRun({
|
|
3867
|
+
store: args.store,
|
|
3868
|
+
reporter: args.reporter,
|
|
3869
|
+
request: args.prepared.request,
|
|
3870
|
+
status: "training",
|
|
3871
|
+
stage: "training",
|
|
3872
|
+
message: "Reusing existing training result.",
|
|
3873
|
+
details: { path: args.prepared.artifacts.trainingReportJson }
|
|
3874
|
+
});
|
|
3875
|
+
const training2 = trainingReportSchema.parse(await readJson(args.prepared.artifacts.trainingReportJson));
|
|
3876
|
+
if (!isDryTraining(training2)) {
|
|
3877
|
+
await modelManifestContract(args.prepared, training2);
|
|
3878
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3879
|
+
await args.store.registerModel({
|
|
3880
|
+
request: args.prepared.request,
|
|
3881
|
+
training: training2,
|
|
3882
|
+
artifactDir: args.prepared.artifacts.runDir
|
|
3883
|
+
});
|
|
3884
|
+
}
|
|
3885
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3886
|
+
return training2;
|
|
3887
|
+
}
|
|
3888
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3889
|
+
await args.store.invalidateRunOutputs(args.prepared.request.run_id, { report: true, model: true });
|
|
3890
|
+
await cleanupStageArtifacts(args.prepared.artifacts, "train");
|
|
3891
|
+
await updateRun({
|
|
3892
|
+
store: args.store,
|
|
3893
|
+
reporter: args.reporter,
|
|
3894
|
+
request: args.prepared.request,
|
|
3895
|
+
status: "training",
|
|
3896
|
+
stage: "training",
|
|
3897
|
+
message: args.config.dryRun ? "Recording dry-run training result." : "Launching local training process.",
|
|
3898
|
+
details: { training_backend: "local-uv", dry_run: args.config.dryRun }
|
|
3899
|
+
});
|
|
3900
|
+
const training = await launchProcessTraining({
|
|
3901
|
+
request: args.prepared.request,
|
|
3902
|
+
artifacts: args.prepared.artifacts,
|
|
3903
|
+
config: args.config,
|
|
3904
|
+
baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
|
|
3905
|
+
reporter: args.runReporter,
|
|
3906
|
+
shouldCancel: () => args.store.isCancellationRequested(args.prepared.request.run_id)
|
|
3907
|
+
});
|
|
3908
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3909
|
+
if (!isDryTraining(training)) {
|
|
3910
|
+
if (!training.model_artifact_uri) {
|
|
3911
|
+
throw new Error("Training process completed without a model_artifact_uri.");
|
|
3912
|
+
}
|
|
3913
|
+
await modelManifestContract(args.prepared, training);
|
|
3914
|
+
}
|
|
3915
|
+
await writeJsonAtomic(args.prepared.artifacts.trainingReportJson, training);
|
|
3916
|
+
await writeStageFingerprint({ stage: "train", prepared: args.prepared, config: args.config });
|
|
3917
|
+
await refreshArtifactManifest(args.prepared);
|
|
3918
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3919
|
+
if (!isDryTraining(training)) {
|
|
3920
|
+
await args.store.registerModel({
|
|
3921
|
+
request: args.prepared.request,
|
|
3922
|
+
training,
|
|
3923
|
+
artifactDir: args.prepared.artifacts.runDir
|
|
3924
|
+
});
|
|
3925
|
+
}
|
|
3926
|
+
return training;
|
|
3927
|
+
}
|
|
3928
|
+
async function runCandidateStage(args) {
|
|
3929
|
+
let verifiedTraining;
|
|
3930
|
+
if (await pathExists2(args.prepared.artifacts.trainingReportJson)) {
|
|
3931
|
+
const trainingCurrent = await canReuseStageArtifact({
|
|
3932
|
+
stage: "train",
|
|
3933
|
+
prepared: args.prepared,
|
|
3934
|
+
config: args.config,
|
|
3935
|
+
verifyModel: true
|
|
3936
|
+
});
|
|
3937
|
+
if (trainingCurrent) {
|
|
3938
|
+
verifiedTraining = trainingReportSchema.parse(
|
|
3939
|
+
await readJson(args.prepared.artifacts.trainingReportJson)
|
|
3940
|
+
);
|
|
3941
|
+
}
|
|
3942
|
+
}
|
|
3943
|
+
if (verifiedTraining && await canReuseStageArtifact({
|
|
3944
|
+
stage: "candidate",
|
|
3945
|
+
prepared: args.prepared,
|
|
3946
|
+
config: args.config,
|
|
3947
|
+
verifyModel: true,
|
|
3948
|
+
additionalPaths: [
|
|
3949
|
+
args.prepared.artifacts.trainingReportJson,
|
|
3950
|
+
stageFingerprintPath(args.prepared, "train"),
|
|
3951
|
+
...generalRegressionRequiredPaths(args.prepared, "candidate")
|
|
3952
|
+
]
|
|
3953
|
+
})) {
|
|
3954
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3955
|
+
await updateRun({
|
|
3956
|
+
store: args.store,
|
|
3957
|
+
reporter: args.reporter,
|
|
3958
|
+
request: args.prepared.request,
|
|
3959
|
+
status: "evaluating_candidate",
|
|
3960
|
+
stage: "evaluating_candidate",
|
|
3961
|
+
message: "Reusing existing candidate evaluation.",
|
|
3962
|
+
details: { path: args.prepared.artifacts.candidateEvalJson }
|
|
3963
|
+
});
|
|
3964
|
+
const training2 = verifiedTraining;
|
|
3965
|
+
if (!isDryTraining(training2)) {
|
|
3966
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3967
|
+
await args.store.registerModel({
|
|
3968
|
+
request: args.prepared.request,
|
|
3969
|
+
training: training2,
|
|
3970
|
+
artifactDir: args.prepared.artifacts.runDir
|
|
3971
|
+
});
|
|
3972
|
+
}
|
|
3973
|
+
const report2 = evalReportSchema.parse(await readJson(args.prepared.artifacts.candidateEvalJson));
|
|
3974
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3975
|
+
return report2;
|
|
3976
|
+
}
|
|
3977
|
+
if (!verifiedTraining) {
|
|
3978
|
+
throw new Error("Candidate evaluation requires current verified training output.");
|
|
3979
|
+
}
|
|
3980
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3981
|
+
await args.store.invalidateRunOutputs(args.prepared.request.run_id, { report: true });
|
|
3982
|
+
await cleanupStageArtifacts(args.prepared.artifacts, "candidate");
|
|
3983
|
+
const training = verifiedTraining;
|
|
3984
|
+
const modelArtifact = training.model_artifact_uri;
|
|
3985
|
+
if (!modelArtifact) throw new Error("candidate stage requires a model_artifact_uri in training-report.json.");
|
|
3986
|
+
if (!isDryTraining(training)) {
|
|
3987
|
+
await modelManifestContract(args.prepared, training);
|
|
3988
|
+
await refreshArtifactManifest(args.prepared);
|
|
3989
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
3990
|
+
await args.store.registerModel({
|
|
3991
|
+
request: args.prepared.request,
|
|
3992
|
+
training,
|
|
3993
|
+
artifactDir: args.prepared.artifacts.runDir
|
|
3994
|
+
});
|
|
3995
|
+
}
|
|
3996
|
+
await updateRun({
|
|
3997
|
+
store: args.store,
|
|
3998
|
+
reporter: args.reporter,
|
|
3999
|
+
request: args.prepared.request,
|
|
4000
|
+
status: "evaluating_candidate",
|
|
4001
|
+
stage: "evaluating_candidate",
|
|
4002
|
+
message: "Running candidate evaluation.",
|
|
4003
|
+
details: { model_artifact_uri: modelArtifact }
|
|
4004
|
+
});
|
|
4005
|
+
const report = await evaluateExamples({
|
|
4006
|
+
kind: "candidate",
|
|
4007
|
+
modelId: modelArtifact,
|
|
4008
|
+
baseModelId: args.prepared.baseModelForEvaluation,
|
|
4009
|
+
baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
|
|
4010
|
+
adapterPath: modelArtifact,
|
|
4011
|
+
examples: args.prepared.examples,
|
|
4012
|
+
system: args.prepared.system,
|
|
4013
|
+
config: args.config,
|
|
4014
|
+
outputPath: args.prepared.artifacts.candidateEvalJson,
|
|
4015
|
+
reporter: args.runReporter,
|
|
4016
|
+
maxExamples: args.prepared.maxEvalExamples,
|
|
4017
|
+
evalSplit: args.prepared.metadata.eval_split,
|
|
4018
|
+
sampleSeed: args.prepared.metadata.eval_sample_seed,
|
|
4019
|
+
shouldCancel: () => args.store.isCancellationRequested(args.prepared.request.run_id)
|
|
4020
|
+
});
|
|
4021
|
+
if (args.prepared.generalRegression) {
|
|
4022
|
+
await updateRun({
|
|
4023
|
+
store: args.store,
|
|
4024
|
+
reporter: args.reporter,
|
|
4025
|
+
request: args.prepared.request,
|
|
4026
|
+
status: "evaluating_candidate",
|
|
4027
|
+
stage: "evaluating_candidate",
|
|
4028
|
+
message: "Running candidate general regression evaluation.",
|
|
4029
|
+
details: {
|
|
4030
|
+
examples: args.prepared.generalRegression.examples.length,
|
|
4031
|
+
dataset: args.prepared.generalRegression.datasetPath
|
|
4032
|
+
}
|
|
4033
|
+
});
|
|
4034
|
+
await evaluateExamples({
|
|
4035
|
+
kind: "candidate",
|
|
4036
|
+
modelId: modelArtifact,
|
|
4037
|
+
baseModelId: args.prepared.baseModelForEvaluation,
|
|
4038
|
+
baseModelRevision: args.config.paths.baseModel ? void 0 : args.prepared.metadata.base_model_revision ?? void 0,
|
|
4039
|
+
adapterPath: modelArtifact,
|
|
4040
|
+
examples: args.prepared.generalRegression.examples,
|
|
4041
|
+
system: args.prepared.generalRegression.system,
|
|
4042
|
+
config: generalRegressionEvaluationConfig(args.config),
|
|
4043
|
+
outputPath: args.prepared.artifacts.generalCandidateEvalJson,
|
|
4044
|
+
reporter: args.runReporter,
|
|
4045
|
+
evalSplit: "general_regression",
|
|
4046
|
+
shouldCancel: () => args.store.isCancellationRequested(args.prepared.request.run_id)
|
|
4047
|
+
});
|
|
4048
|
+
}
|
|
4049
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
4050
|
+
await writeStageFingerprint({
|
|
4051
|
+
stage: "candidate",
|
|
4052
|
+
prepared: args.prepared,
|
|
4053
|
+
config: args.config,
|
|
4054
|
+
training
|
|
4055
|
+
});
|
|
4056
|
+
return report;
|
|
4057
|
+
}
|
|
4058
|
+
async function runReportStage(args) {
|
|
4059
|
+
if (!await pathExists2(args.prepared.artifacts.baselineEvalJson)) {
|
|
4060
|
+
throw new Error("Run reporting requires baseline-eval.json.");
|
|
4061
|
+
}
|
|
4062
|
+
if (!await pathExists2(args.prepared.artifacts.candidateEvalJson)) {
|
|
4063
|
+
throw new Error("Run reporting requires candidate-eval.json.");
|
|
4064
|
+
}
|
|
4065
|
+
if (!await pathExists2(args.prepared.artifacts.trainingReportJson)) {
|
|
4066
|
+
throw new Error("Run reporting requires training-report.json.");
|
|
4067
|
+
}
|
|
4068
|
+
if (args.prepared.generalRegression && (!await pathExists2(args.prepared.artifacts.generalBaselineEvalJson) || !await pathExists2(args.prepared.artifacts.generalCandidateEvalJson))) {
|
|
4069
|
+
throw new Error("Run reporting requires both general regression evaluations.");
|
|
4070
|
+
}
|
|
4071
|
+
const currentTraining = await canReuseStageArtifact({
|
|
4072
|
+
stage: "train",
|
|
4073
|
+
prepared: args.prepared,
|
|
4074
|
+
config: args.config,
|
|
4075
|
+
verifyModel: true
|
|
4076
|
+
});
|
|
4077
|
+
const currentBaseline = await canReuseStageArtifact({
|
|
4078
|
+
stage: "baseline",
|
|
4079
|
+
prepared: args.prepared,
|
|
4080
|
+
config: args.config,
|
|
4081
|
+
additionalPaths: generalRegressionRequiredPaths(args.prepared, "baseline")
|
|
4082
|
+
});
|
|
4083
|
+
const currentCandidate = await canReuseStageArtifact({
|
|
4084
|
+
stage: "candidate",
|
|
4085
|
+
prepared: args.prepared,
|
|
4086
|
+
config: args.config,
|
|
4087
|
+
verifyModel: true,
|
|
4088
|
+
additionalPaths: [
|
|
4089
|
+
args.prepared.artifacts.trainingReportJson,
|
|
4090
|
+
stageFingerprintPath(args.prepared, "train"),
|
|
4091
|
+
...generalRegressionRequiredPaths(args.prepared, "candidate")
|
|
4092
|
+
]
|
|
4093
|
+
});
|
|
4094
|
+
if (!currentTraining || !currentBaseline || !currentCandidate) {
|
|
4095
|
+
throw new Error("report stage inputs are stale for the current request/config. Re-run baseline, train, and candidate as needed.");
|
|
4096
|
+
}
|
|
4097
|
+
await verifyReusableArtifacts(args.prepared.artifacts, [
|
|
4098
|
+
args.prepared.artifacts.baselineEvalJson,
|
|
4099
|
+
stageFingerprintPath(args.prepared, "baseline"),
|
|
4100
|
+
args.prepared.artifacts.candidateEvalJson,
|
|
4101
|
+
stageFingerprintPath(args.prepared, "candidate"),
|
|
4102
|
+
args.prepared.artifacts.trainingReportJson,
|
|
4103
|
+
stageFingerprintPath(args.prepared, "train"),
|
|
4104
|
+
...generalRegressionRequiredPaths(args.prepared, "baseline"),
|
|
4105
|
+
...generalRegressionRequiredPaths(args.prepared, "candidate")
|
|
4106
|
+
], { verifyModel: true });
|
|
4107
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
4108
|
+
await args.store.invalidateRunOutputs(args.prepared.request.run_id, { report: true });
|
|
4109
|
+
await cleanupStageArtifacts(args.prepared.artifacts, "report");
|
|
4110
|
+
await updateRun({
|
|
4111
|
+
store: args.store,
|
|
4112
|
+
reporter: args.reporter,
|
|
4113
|
+
request: args.prepared.request,
|
|
4114
|
+
status: "reporting",
|
|
4115
|
+
stage: "reporting",
|
|
4116
|
+
message: "Writing run report.",
|
|
4117
|
+
details: { report_path: args.prepared.artifacts.runReportJson }
|
|
4118
|
+
});
|
|
4119
|
+
const baseline = evalReportSchema.parse(await readJson(args.prepared.artifacts.baselineEvalJson));
|
|
4120
|
+
const candidate = evalReportSchema.parse(await readJson(args.prepared.artifacts.candidateEvalJson));
|
|
4121
|
+
const training = trainingReportSchema.parse(await readJson(args.prepared.artifacts.trainingReportJson));
|
|
4122
|
+
const comparison = compareEvalReports(baseline, candidate);
|
|
4123
|
+
let generalRegression;
|
|
4124
|
+
if (args.prepared.generalRegression) {
|
|
4125
|
+
const generalBaseline = evalReportSchema.parse(
|
|
4126
|
+
await readJson(args.prepared.artifacts.generalBaselineEvalJson)
|
|
4127
|
+
);
|
|
4128
|
+
const generalCandidate = evalReportSchema.parse(
|
|
4129
|
+
await readJson(args.prepared.artifacts.generalCandidateEvalJson)
|
|
4130
|
+
);
|
|
4131
|
+
const generalComparison = compareEvalReports(generalBaseline, generalCandidate);
|
|
4132
|
+
const policy = args.config.evaluation.generalRegression;
|
|
4133
|
+
const gate = evaluateGeneralRegressionGate(generalComparison, policy);
|
|
4134
|
+
generalRegression = {
|
|
4135
|
+
dataset_uri: fileUri(args.prepared.generalRegression.datasetPath),
|
|
4136
|
+
dataset_sha256: args.prepared.generalRegression.datasetSha256,
|
|
4137
|
+
baseline: generalBaseline,
|
|
4138
|
+
candidate: generalCandidate,
|
|
4139
|
+
comparison: generalComparison,
|
|
4140
|
+
policy: {
|
|
4141
|
+
max_score_drop: policy.maxScoreDrop,
|
|
4142
|
+
max_pass_rate_drop: policy.maxPassRateDrop
|
|
4143
|
+
},
|
|
4144
|
+
passed: gate.passed,
|
|
4145
|
+
failures: gate.failures
|
|
4146
|
+
};
|
|
4147
|
+
}
|
|
4148
|
+
const completedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4149
|
+
const duration = elapsed(args.startedPerf);
|
|
4150
|
+
const report = runReportSchema.parse({
|
|
4151
|
+
run_id: args.prepared.request.run_id,
|
|
4152
|
+
behavior_spec_id: args.prepared.request.behavior_spec_id,
|
|
4153
|
+
user_id: args.prepared.request.user_id,
|
|
4154
|
+
run_number: args.prepared.request.run_number,
|
|
4155
|
+
base_model: args.prepared.request.spec_snapshot.base_model,
|
|
4156
|
+
fine_tuned_model_id: training.model_artifact_uri ?? training.training_job_name,
|
|
4157
|
+
status: "completed",
|
|
4158
|
+
baseline,
|
|
4159
|
+
candidate,
|
|
4160
|
+
comparison,
|
|
4161
|
+
general_regression: generalRegression,
|
|
4162
|
+
training,
|
|
4163
|
+
artifact_uris: {
|
|
4164
|
+
dataset: fileUri(args.prepared.artifacts.trainingJsonl),
|
|
4165
|
+
baseline_eval: fileUri(args.prepared.artifacts.baselineEvalJson),
|
|
4166
|
+
candidate_eval: fileUri(args.prepared.artifacts.candidateEvalJson),
|
|
4167
|
+
general_baseline_eval: generalRegression ? fileUri(args.prepared.artifacts.generalBaselineEvalJson) : void 0,
|
|
4168
|
+
general_candidate_eval: generalRegression ? fileUri(args.prepared.artifacts.generalCandidateEvalJson) : void 0,
|
|
4169
|
+
report: fileUri(args.prepared.artifacts.runReportJson)
|
|
4170
|
+
},
|
|
4171
|
+
run_metadata: {
|
|
4172
|
+
base_model: args.prepared.request.spec_snapshot.base_model,
|
|
4173
|
+
fine_tuned_model_id: training.model_artifact_uri ?? training.training_job_name,
|
|
4174
|
+
dataset_prebuilt: args.prepared.metadata.dataset_prebuilt,
|
|
4175
|
+
dataset_format: args.prepared.metadata.dataset_format,
|
|
4176
|
+
dataset_uri: fileUri(args.prepared.artifacts.trainingJsonl),
|
|
4177
|
+
spec_example_count: args.prepared.request.spec_snapshot.examples.length,
|
|
4178
|
+
training_example_count: args.prepared.metadata.training_example_count,
|
|
4179
|
+
eval_examples_total: baseline.eval_examples_total,
|
|
4180
|
+
eval_examples_used: baseline.eval_examples_used,
|
|
4181
|
+
eval_split: baseline.eval_split,
|
|
4182
|
+
eval_sample_seed: baseline.eval_sample_seed ?? null,
|
|
4183
|
+
started_at: args.startedAt,
|
|
4184
|
+
completed_at: completedAt,
|
|
4185
|
+
elapsed_ms: duration.ms,
|
|
4186
|
+
elapsed_seconds: duration.seconds
|
|
4187
|
+
},
|
|
4188
|
+
created_at: completedAt
|
|
4189
|
+
});
|
|
4190
|
+
await writeJsonAtomic(args.prepared.artifacts.runReportJson, report);
|
|
4191
|
+
await refreshArtifactManifest(args.prepared);
|
|
4192
|
+
await throwIfCancelled(args.store, args.prepared.request);
|
|
4193
|
+
const completedState = await args.store.completeRun(
|
|
4194
|
+
report,
|
|
4195
|
+
args.prepared.artifacts.runDir,
|
|
4196
|
+
args.prepared.artifacts.runReportJson
|
|
4197
|
+
);
|
|
4198
|
+
if (completedState.status === "cancelled") {
|
|
4199
|
+
throw new ProcessCancelledError(`Run ${args.prepared.request.run_id} was cancelled.`);
|
|
4200
|
+
}
|
|
4201
|
+
await args.reporter?.onEvent?.({
|
|
4202
|
+
stage: "completed",
|
|
4203
|
+
status: "completed",
|
|
4204
|
+
message: "Run completed successfully.",
|
|
4205
|
+
details: {
|
|
4206
|
+
report_path: args.prepared.artifacts.runReportJson,
|
|
4207
|
+
...!isDryTraining(training) ? { model_id: `local-${args.prepared.request.run_id}` } : {},
|
|
4208
|
+
avg_score_delta: comparison.avg_score_delta,
|
|
4209
|
+
general_regression_passed: generalRegression?.passed,
|
|
4210
|
+
elapsed_seconds: duration.seconds
|
|
4211
|
+
}
|
|
4212
|
+
});
|
|
4213
|
+
return report;
|
|
4214
|
+
}
|
|
4215
|
+
async function runLocalPipeline(input) {
|
|
4216
|
+
const pipeline = validateLocalPipeline(input.pipeline);
|
|
4217
|
+
const validated = await validateLocalFineTuneInput({
|
|
4218
|
+
request: input.request,
|
|
4219
|
+
config: input.config
|
|
4220
|
+
});
|
|
4221
|
+
input = { ...input, ...validated };
|
|
4222
|
+
const startedPerf = performance.now();
|
|
4223
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
4224
|
+
const prefix = artifactPrefix(input.request);
|
|
4225
|
+
const artifacts = resolveRunArtifacts({ artifactRoot: input.config.artifactRoot, prefix });
|
|
4226
|
+
const store = createLocalStore(input.config.storeRoot);
|
|
4227
|
+
const releaseRunLock = await acquireRunLock(store, input.request.run_id);
|
|
4228
|
+
let releaseArtifactLock;
|
|
4229
|
+
try {
|
|
4230
|
+
await claimRunArtifactDirectory({
|
|
4231
|
+
artifacts,
|
|
4232
|
+
runId: input.request.run_id,
|
|
4233
|
+
userId: input.request.user_id,
|
|
4234
|
+
behaviorSpecId: input.request.behavior_spec_id
|
|
4235
|
+
});
|
|
4236
|
+
releaseArtifactLock = await acquireArtifactLock(artifacts);
|
|
4237
|
+
await ensureRunRecord({ request: input.request, artifacts, store, reporter: input.reporter });
|
|
4238
|
+
const runReporter = createStoreReporter({ request: input.request, store, reporter: input.reporter });
|
|
4239
|
+
let prepared;
|
|
4240
|
+
try {
|
|
4241
|
+
await throwIfCancelled(store, input.request);
|
|
4242
|
+
prepared = await prepareStage({
|
|
4243
|
+
request: input.request,
|
|
4244
|
+
config: input.config,
|
|
4245
|
+
artifacts,
|
|
4246
|
+
store,
|
|
4247
|
+
reporter: input.reporter
|
|
4248
|
+
});
|
|
4249
|
+
const outputs = {};
|
|
4250
|
+
for (const step of pipeline.steps) {
|
|
4251
|
+
await throwIfCancelled(store, input.request);
|
|
4252
|
+
if (step.uses === "train") {
|
|
4253
|
+
const training = await runTrainStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
|
|
4254
|
+
outputs[step.id] = { training, modelArtifactUri: training.model_artifact_uri };
|
|
4255
|
+
await refreshArtifactManifest(prepared);
|
|
4256
|
+
continue;
|
|
4257
|
+
}
|
|
4258
|
+
if (step.uses === "evaluate") {
|
|
4259
|
+
const isBase = step.with.model === "base";
|
|
4260
|
+
const report = isBase ? await runBaselineStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter }) : await runCandidateStage({ prepared, config: input.config, store, reporter: input.reporter, runReporter });
|
|
4261
|
+
const reportPath = join8(artifacts.runDir, "evaluations", `${step.id}.json`);
|
|
4262
|
+
await writeJsonAtomic(reportPath, { ...report, artifact_uri: fileUri(reportPath) });
|
|
4263
|
+
outputs[step.id] = { report: { ...report, artifact_uri: fileUri(reportPath) }, reportPath };
|
|
4264
|
+
await refreshArtifactManifest(prepared);
|
|
4265
|
+
continue;
|
|
4266
|
+
}
|
|
4267
|
+
const before = outputs[referenceParts(step.with.before)[0]]?.report;
|
|
4268
|
+
const after = outputs[referenceParts(step.with.after)[0]]?.report;
|
|
4269
|
+
if (!before || !after) throw new Error(`Local compare step ${step.id} is missing validated evaluation outputs.`);
|
|
4270
|
+
outputs[step.id] = { comparison: compareEvalReports(before, after) };
|
|
4271
|
+
}
|
|
4272
|
+
const compatibilityRecipe = pipeline.steps.length === 4 && pipeline.steps.map((step) => `${step.id}:${step.uses}`).join(",") === "baseline:evaluate,train:train,candidate:evaluate,compare:compare";
|
|
4273
|
+
if (compatibilityRecipe) {
|
|
4274
|
+
const report = await runReportStage({
|
|
4275
|
+
prepared,
|
|
4276
|
+
config: input.config,
|
|
4277
|
+
store,
|
|
4278
|
+
reporter: input.reporter,
|
|
4279
|
+
startedAt,
|
|
4280
|
+
startedPerf
|
|
4281
|
+
});
|
|
4282
|
+
return { request: input.request, pipeline, status: "completed", outputs, report, reportPath: artifacts.runReportJson, artifactDir: artifacts.runDir };
|
|
4283
|
+
}
|
|
4284
|
+
await store.updateRun({
|
|
4285
|
+
runId: input.request.run_id,
|
|
4286
|
+
status: "stage_completed",
|
|
4287
|
+
stage: "stage_completed",
|
|
4288
|
+
message: "Pipeline steps completed; no full compatibility report was requested.",
|
|
4289
|
+
details: { completed_steps: pipeline.steps.map((step) => step.id) }
|
|
4290
|
+
});
|
|
4291
|
+
await input.reporter?.onEvent?.({
|
|
4292
|
+
stage: "stage_completed",
|
|
4293
|
+
status: "stage_completed",
|
|
4294
|
+
message: "Pipeline steps completed without a full compatibility report."
|
|
4295
|
+
});
|
|
4296
|
+
return { request: input.request, pipeline, status: "stage_completed", outputs, artifactDir: artifacts.runDir };
|
|
4297
|
+
} catch (error) {
|
|
4298
|
+
const cancelled = error instanceof ProcessCancelledError || await store.isCancellationRequested(input.request.run_id).catch(() => false);
|
|
4299
|
+
if (cancelled) {
|
|
4300
|
+
const state = await store.getRun(input.request.run_id).catch(() => null);
|
|
4301
|
+
if (state?.status !== "cancelled") await store.cancelRun(input.request.run_id).catch(() => void 0);
|
|
4302
|
+
await store.finalizeCancellation(input.request.run_id).catch(() => void 0);
|
|
4303
|
+
await input.reporter?.onEvent?.({
|
|
4304
|
+
stage: "cancelled",
|
|
4305
|
+
status: "cancelled",
|
|
4306
|
+
message: "Run cancelled."
|
|
4307
|
+
});
|
|
4308
|
+
} else {
|
|
4309
|
+
await store.failRun(input.request.run_id, error instanceof Error ? error.message : String(error)).catch(() => void 0);
|
|
4310
|
+
await input.reporter?.onEvent?.({
|
|
4311
|
+
stage: "failed",
|
|
4312
|
+
status: "failed",
|
|
4313
|
+
message: error instanceof Error ? error.message : String(error)
|
|
4314
|
+
});
|
|
4315
|
+
}
|
|
4316
|
+
throw error;
|
|
4317
|
+
}
|
|
4318
|
+
} finally {
|
|
4319
|
+
await releaseArtifactLock?.();
|
|
4320
|
+
await releaseRunLock();
|
|
4321
|
+
}
|
|
4322
|
+
}
|
|
4323
|
+
async function runLocalFineTune(input) {
|
|
4324
|
+
const result = await runLocalPipeline({ ...input, pipeline: canonicalLocalPipeline() });
|
|
4325
|
+
if (!result.report || !result.reportPath) {
|
|
4326
|
+
throw new Error("Canonical local pipeline completed without a compatibility report.");
|
|
4327
|
+
}
|
|
4328
|
+
return {
|
|
4329
|
+
request: result.request,
|
|
4330
|
+
report: result.report,
|
|
4331
|
+
reportPath: result.reportPath,
|
|
4332
|
+
artifactDir: result.artifactDir
|
|
4333
|
+
};
|
|
4334
|
+
}
|
|
4335
|
+
|
|
4336
|
+
export {
|
|
4337
|
+
writeJsonAtomic,
|
|
4338
|
+
assertArtifactManifest,
|
|
4339
|
+
resolveTrainingModel,
|
|
4340
|
+
assertUsableModelArtifact,
|
|
4341
|
+
datasetFormatSchema,
|
|
4342
|
+
baseModelRevisionSchema,
|
|
4343
|
+
behaviorSpecExampleSchema,
|
|
4344
|
+
specSnapshotSchema,
|
|
4345
|
+
datasetPrebuiltSchema,
|
|
4346
|
+
fineTuneHyperparametersSchema,
|
|
4347
|
+
fineTuneRunRequestSchema,
|
|
4348
|
+
localBehaviorSpecFileSchema,
|
|
4349
|
+
evalExampleResultSchema,
|
|
4350
|
+
jsonFieldMetricsSchema,
|
|
4351
|
+
evalSplitSchema,
|
|
4352
|
+
evalReportSchema,
|
|
4353
|
+
comparisonReportSchema,
|
|
4354
|
+
trainingReportSchema,
|
|
4355
|
+
runReportSchema,
|
|
4356
|
+
localRunnerConfigSchema,
|
|
4357
|
+
DEFAULT_LOCAL_SPEC_PATH,
|
|
4358
|
+
resolveLocalRunInputPaths,
|
|
4359
|
+
initLocalSpecFile,
|
|
4360
|
+
initLocalRunnerConfigFile,
|
|
4361
|
+
generatedPlaceholderIssues,
|
|
4362
|
+
assertLocalRunInputReady,
|
|
4363
|
+
runRequestFromLocalSpec,
|
|
4364
|
+
loadLocalRunInput,
|
|
4365
|
+
normalizeChatJsonlForRelocation,
|
|
4366
|
+
buildSystemMessage,
|
|
4367
|
+
exampleToChatRow,
|
|
4368
|
+
compileSpecToJsonl,
|
|
4369
|
+
examplesFromSpec,
|
|
4370
|
+
examplesFromChatJsonl,
|
|
4371
|
+
evaluationSuiteFromChatJsonl,
|
|
4372
|
+
reportInBackground,
|
|
4373
|
+
sanitizeLogLine,
|
|
4374
|
+
forwardStreamLines,
|
|
4375
|
+
localRuntimePackageRoot,
|
|
4376
|
+
buildBundledPythonCommand,
|
|
4377
|
+
withBundledPythonEnvironment,
|
|
4378
|
+
isTerminalRunState,
|
|
4379
|
+
defaultLocalHome,
|
|
4380
|
+
localStorePaths,
|
|
4381
|
+
createLocalStore,
|
|
4382
|
+
minimalMachineLearningEnvironment,
|
|
4383
|
+
withHuggingFaceCacheEnvironment,
|
|
4384
|
+
withOfflineHuggingFaceCacheEnvironment,
|
|
4385
|
+
tokenF1,
|
|
4386
|
+
buildModelPrefetchPayload,
|
|
4387
|
+
verifyLocalBaseModel,
|
|
4388
|
+
prefetchBaseModel,
|
|
4389
|
+
evaluateGeneralRegressionGate,
|
|
4390
|
+
canonicalLocalPipeline,
|
|
4391
|
+
validateLocalPipeline,
|
|
4392
|
+
loadJsonFile,
|
|
4393
|
+
loadRunRequest,
|
|
4394
|
+
loadLocalRunnerConfig,
|
|
4395
|
+
validateLocalFineTuneInput,
|
|
4396
|
+
fingerprintLocalBaseModel,
|
|
4397
|
+
runLocalPipeline,
|
|
4398
|
+
runLocalFineTune
|
|
4399
|
+
};
|
|
4400
|
+
//# sourceMappingURL=chunk-DAP2Z7EZ.js.map
|