@workbench-ai/workbench 0.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter-project.d.ts +29 -0
- package/dist/adapter-project.d.ts.map +1 -0
- package/dist/adapter-project.js +363 -0
- package/dist/benchmark-fingerprint.d.ts +6 -0
- package/dist/benchmark-fingerprint.d.ts.map +1 -0
- package/dist/benchmark-fingerprint.js +101 -0
- package/dist/command-model.d.ts +5 -0
- package/dist/command-model.d.ts.map +1 -0
- package/dist/command-model.js +558 -0
- package/dist/dev-open/client.css +8157 -0
- package/dist/dev-open/client.js +252596 -0
- package/dist/dev-open/fonts/geist-cyrillic-wght-normal.woff2 +0 -0
- package/dist/dev-open/fonts/geist-latin-ext-wght-normal.woff2 +0 -0
- package/dist/dev-open/fonts/geist-latin-wght-normal.woff2 +0 -0
- package/dist/dev-open-server.d.ts +57 -0
- package/dist/dev-open-server.d.ts.map +1 -0
- package/dist/dev-open-server.js +496 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3943 -0
- package/dist/init-scaffold.d.ts +22 -0
- package/dist/init-scaffold.d.ts.map +1 -0
- package/dist/init-scaffold.js +30 -0
- package/dist/init-template-pack.d.ts +19 -0
- package/dist/init-template-pack.d.ts.map +1 -0
- package/dist/init-template-pack.js +250 -0
- package/dist/local-archive.d.ts +23 -0
- package/dist/local-archive.d.ts.map +1 -0
- package/dist/local-archive.js +741 -0
- package/dist/project-source.d.ts +51 -0
- package/dist/project-source.d.ts.map +1 -0
- package/dist/project-source.js +700 -0
- package/dist/workbench.d.ts +3 -0
- package/dist/workbench.d.ts.map +1 -0
- package/dist/workbench.js +4 -0
- package/dist/workspace-snapshot.d.ts +10 -0
- package/dist/workspace-snapshot.d.ts.map +1 -0
- package/dist/workspace-snapshot.js +81 -0
- package/package.json +45 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,3943 @@
|
|
|
1
|
+
import { promises as fs } from "node:fs";
|
|
2
|
+
import { spawn } from "node:child_process";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { createRequire } from "node:module";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { Writable } from "node:stream";
|
|
8
|
+
import { createSubjectFilePreview, createBaselineSubjectJob as createRuntimeBaselineSubjectJob, executeWorkbenchExecutionJob, engineResolveBindingForSpec, filterSubjectSourceFiles, workbenchExecutionPurpose, createWorkbenchAdapterAuthBundle, createSubjectRevisionTraceInputFiles, DOCKER_SANDBOX_BACKEND, localWorkbenchAdapterAuthStore, materializeWorkbenchRunResult, normalizeSurfaceFiles, planWorkbenchExecutionJobsForPurpose, runWorkbenchExecutionDag, resolveEngineCaseExecutionConfig, resolveWorkbenchResolvedSourceYaml, summarizeSubjectFiles, validateWorkbenchRunEnvelope, parseWorkbenchAdapterAuthTarget, } from "@workbench-ai/workbench-core";
|
|
9
|
+
import { assertWorkbenchAdapterOperationResultOk, collectWorkbenchAdapterAuthRequirements, WORKBENCH_ADAPTER_RESULT_FILE, WORKBENCH_ADAPTER_RESULT_PROTOCOL, normalizeWorkbenchAdapterOperationRequest, readWorkbenchAdapterOperationResult, workbenchAdapterOperationCommand, workbenchAdapterOperationResultPath, withDefaultWorkbenchAdapterAuthProfiles as applyDefaultWorkbenchAdapterAuthProfiles, } from "@workbench-ai/workbench-protocol";
|
|
10
|
+
import { commandUsage, HOSTED_WATCH_LIFECYCLE_NOTE, LOCAL_DEV_OPEN_LIFECYCLE_NOTE, rootUsage, } from "./command-model.js";
|
|
11
|
+
import { startLocalWorkbenchDevServer } from "./dev-open-server.js";
|
|
12
|
+
import { createWorkbenchInitScaffold, } from "./init-scaffold.js";
|
|
13
|
+
import { defaultAdapterManifests, composeRuntimeDockerfileWithAdapters, resolveDefaultWorkbenchAdapter, resolveProjectAdapterSource, resolveWorkbenchAdaptersForProject, WORKBENCH_ADAPTER_MANIFEST_FILE, } from "./adapter-project.js";
|
|
14
|
+
import { appendLocalRun, loadLocalArchive, materializeSubjectRoot, readLocalSubject, readLocalSubjectFiles, saveLocalArchive, saveLocalJobs, setLocalActive, upsertLocalSubject, upsertLocalEvaluation, } from "./local-archive.js";
|
|
15
|
+
import { readSnapshotFiles, WorkspaceSnapshotError, } from "./workspace-snapshot.js";
|
|
16
|
+
import { readLocalProjectSource, WORKBENCH_BENCHMARK_FILE, } from "./project-source.js";
|
|
17
|
+
import { localBenchmarkFingerprint, localSubjectFingerprint, } from "./benchmark-fingerprint.js";
|
|
18
|
+
const require = createRequire(import.meta.url);
|
|
19
|
+
function getCliVersion() {
|
|
20
|
+
const manifest = require("../package.json");
|
|
21
|
+
return typeof manifest.version === "string" ? manifest.version : "unknown";
|
|
22
|
+
}
|
|
23
|
+
const DEFAULT_BASE_URL = "https://v2.workbench.ai";
|
|
24
|
+
export async function runCli(argv, io = {
|
|
25
|
+
stdin: process.stdin,
|
|
26
|
+
stdout: process.stdout,
|
|
27
|
+
stderr: process.stderr,
|
|
28
|
+
}, runtimeOptions = {}) {
|
|
29
|
+
try {
|
|
30
|
+
if (argv.length === 0 || argv[0] === "--help" || argv[0] === "-h") {
|
|
31
|
+
io.stdout.write(`${rootUsage}\n`);
|
|
32
|
+
return 0;
|
|
33
|
+
}
|
|
34
|
+
if (argv[0] === "--version" || argv[0] === "-v") {
|
|
35
|
+
io.stdout.write(`workbench ${getCliVersion()}\n`);
|
|
36
|
+
return 0;
|
|
37
|
+
}
|
|
38
|
+
if (argv.includes("--help") || argv.includes("-h")) {
|
|
39
|
+
const commandPath = commandPathForHelp(argv);
|
|
40
|
+
const usage = commandUsage(commandPath);
|
|
41
|
+
if (!usage) {
|
|
42
|
+
throw new UsageError(`Unknown command: ${commandPath || argv.join(" ")}`);
|
|
43
|
+
}
|
|
44
|
+
io.stdout.write(`${usage}\n`);
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
if (argv[0] === "init") {
|
|
48
|
+
return await localInit(argv.slice(1), io);
|
|
49
|
+
}
|
|
50
|
+
if (argv[0] === "check") {
|
|
51
|
+
return await localValidate(argv.slice(1), io);
|
|
52
|
+
}
|
|
53
|
+
if (argv[0] === "login") {
|
|
54
|
+
return await login(argv.slice(1), io);
|
|
55
|
+
}
|
|
56
|
+
if (argv[0] === "logout") {
|
|
57
|
+
return await logout(argv.slice(1), io);
|
|
58
|
+
}
|
|
59
|
+
if (argv[0] === "whoami") {
|
|
60
|
+
return await authStatus(argv.slice(1), io);
|
|
61
|
+
}
|
|
62
|
+
if (argv[0] === "clone") {
|
|
63
|
+
return await cloneProject(argv.slice(1), io);
|
|
64
|
+
}
|
|
65
|
+
if (argv[0] === "fetch") {
|
|
66
|
+
return await fetchProject(argv.slice(1), io);
|
|
67
|
+
}
|
|
68
|
+
if (argv[0] === "pull") {
|
|
69
|
+
return await pullProject(argv.slice(1), io);
|
|
70
|
+
}
|
|
71
|
+
if (argv[0] === "push") {
|
|
72
|
+
return await pushBenchmark(argv.slice(1), io);
|
|
73
|
+
}
|
|
74
|
+
if (argv[0] === "remote") {
|
|
75
|
+
return await runRemoteCommand(argv.slice(1), io);
|
|
76
|
+
}
|
|
77
|
+
if (argv[0] === "eval") {
|
|
78
|
+
return await localEvaluateSubject(argv.slice(1), io, runtimeOptions);
|
|
79
|
+
}
|
|
80
|
+
if (argv[0] === "improve") {
|
|
81
|
+
return await localRun(argv.slice(1), io, runtimeOptions);
|
|
82
|
+
}
|
|
83
|
+
if (argv[0] === "checkpoint") {
|
|
84
|
+
return await localCheckpoint(argv.slice(1), io);
|
|
85
|
+
}
|
|
86
|
+
if (argv[0] === "restore") {
|
|
87
|
+
return await localRestore(argv.slice(1), io);
|
|
88
|
+
}
|
|
89
|
+
if (argv[0] === "open") {
|
|
90
|
+
return await localDevOpen(argv.slice(1), io);
|
|
91
|
+
}
|
|
92
|
+
if (argv[0] === "auth") {
|
|
93
|
+
return await runAuthCommand(argv.slice(1), io);
|
|
94
|
+
}
|
|
95
|
+
if (argv[0] === "adapters") {
|
|
96
|
+
return await runAdaptersCommand(argv.slice(1), io);
|
|
97
|
+
}
|
|
98
|
+
if (argv[0] === "cloud") {
|
|
99
|
+
return await runCloudCommand(argv.slice(1), io);
|
|
100
|
+
}
|
|
101
|
+
const commandPath = argv.slice(0, 2).join(" ");
|
|
102
|
+
const rest = argv.slice(2);
|
|
103
|
+
switch (commandPath) {
|
|
104
|
+
case "runs list":
|
|
105
|
+
return await localRunList(rest, io);
|
|
106
|
+
case "runs show":
|
|
107
|
+
return await localRunShow(rest, io);
|
|
108
|
+
case "subjects list":
|
|
109
|
+
return await localSubjectList(rest, io);
|
|
110
|
+
case "subjects show":
|
|
111
|
+
return await localSubjectShow(rest, io);
|
|
112
|
+
case "subjects files":
|
|
113
|
+
return await localSubjectFiles(rest, io);
|
|
114
|
+
case "subjects preview":
|
|
115
|
+
return await localSubjectPreview(rest, io);
|
|
116
|
+
default:
|
|
117
|
+
break;
|
|
118
|
+
}
|
|
119
|
+
throw new UsageError(`Unknown command: ${argv.join(" ")}`);
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
const jsonRequested = argv.includes("--json");
|
|
123
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
124
|
+
if (jsonRequested) {
|
|
125
|
+
io.stdout.write(`${JSON.stringify({ ok: false, error: message }, null, 2)}\n`);
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
io.stderr.write(`${message}\n\n${rootUsage}\n`);
|
|
129
|
+
}
|
|
130
|
+
return error instanceof UsageError || error instanceof WorkspaceSnapshotError ? 2 : 1;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
function commandPathForHelp(argv) {
|
|
134
|
+
const positionals = argv.filter((arg) => arg !== "--help" && arg !== "-h" && !arg.startsWith("--"));
|
|
135
|
+
if (positionals[0] === "cloud") {
|
|
136
|
+
return positionals.slice(0, 3).join(" ");
|
|
137
|
+
}
|
|
138
|
+
if (positionals[0] === "adapters" && positionals[1] === "test") {
|
|
139
|
+
return "adapters test";
|
|
140
|
+
}
|
|
141
|
+
if (positionals[0] === "auth" || positionals[0] === "remote") {
|
|
142
|
+
return positionals.slice(0, 2).join(" ");
|
|
143
|
+
}
|
|
144
|
+
if (positionals[0] === "runs" &&
|
|
145
|
+
["list", "show"].includes(positionals[1] ?? "")) {
|
|
146
|
+
return positionals.slice(0, 2).join(" ");
|
|
147
|
+
}
|
|
148
|
+
if (positionals[0] === "subjects" &&
|
|
149
|
+
["list", "show", "files", "preview"].includes(positionals[1] ?? "")) {
|
|
150
|
+
return positionals.slice(0, 2).join(" ");
|
|
151
|
+
}
|
|
152
|
+
return positionals[0] ?? "";
|
|
153
|
+
}
|
|
154
|
+
async function runCloudCommand(argv, io) {
|
|
155
|
+
const command = argv[0];
|
|
156
|
+
const rest = argv.slice(1);
|
|
157
|
+
switch (command) {
|
|
158
|
+
case "eval":
|
|
159
|
+
return await startHostedWorkflow("eval", rest, io);
|
|
160
|
+
case "improve":
|
|
161
|
+
return await startHostedWorkflow("improve", rest, io);
|
|
162
|
+
case "open":
|
|
163
|
+
return await openWorkbench(rest, io);
|
|
164
|
+
case "watch":
|
|
165
|
+
return await runWatch(rest, io);
|
|
166
|
+
case "logs":
|
|
167
|
+
return await runLogs(rest, io);
|
|
168
|
+
case "fork":
|
|
169
|
+
return await forkProject(rest, io);
|
|
170
|
+
case "star":
|
|
171
|
+
return await starProject(rest, io, true);
|
|
172
|
+
case "unstar":
|
|
173
|
+
return await starProject(rest, io, false);
|
|
174
|
+
default:
|
|
175
|
+
break;
|
|
176
|
+
}
|
|
177
|
+
const commandPath = argv.slice(0, 2).join(" ");
|
|
178
|
+
const subRest = argv.slice(2);
|
|
179
|
+
switch (commandPath) {
|
|
180
|
+
case "benchmarks list":
|
|
181
|
+
return await benchmarkList(subRest, io);
|
|
182
|
+
case "benchmarks show":
|
|
183
|
+
return await benchmarkShow(subRest, io);
|
|
184
|
+
case "benchmarks versions":
|
|
185
|
+
return await benchmarkVersions(subRest, io);
|
|
186
|
+
case "benchmarks starred":
|
|
187
|
+
return await benchmarkStarred(subRest, io);
|
|
188
|
+
case "benchmarks delete":
|
|
189
|
+
return await benchmarkDelete(subRest, io);
|
|
190
|
+
case "runs list":
|
|
191
|
+
return await runList(subRest, io);
|
|
192
|
+
case "runs show":
|
|
193
|
+
return await runShow(subRest, io);
|
|
194
|
+
case "runs cancel":
|
|
195
|
+
return await runCancel(subRest, io);
|
|
196
|
+
case "subjects list":
|
|
197
|
+
return await subjectList(subRest, io);
|
|
198
|
+
case "subjects show":
|
|
199
|
+
return await subjectShow(subRest, io);
|
|
200
|
+
case "subjects files":
|
|
201
|
+
return await subjectFiles(subRest, io);
|
|
202
|
+
case "subjects preview":
|
|
203
|
+
return await subjectPreview(subRest, io);
|
|
204
|
+
case "subjects pull":
|
|
205
|
+
return await subjectExport(subRest, io);
|
|
206
|
+
case "subjects publish":
|
|
207
|
+
return await subjectVisibility(subRest, io, "public");
|
|
208
|
+
case "subjects unpublish":
|
|
209
|
+
return await subjectVisibility(subRest, io, "private");
|
|
210
|
+
default:
|
|
211
|
+
throw new UsageError(`Unknown command: cloud ${argv.join(" ")}`);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
async function localDevOpen(argv, io) {
|
|
215
|
+
const parsed = parseArgs(argv);
|
|
216
|
+
rejectUnknownFlags(parsed, new Set(["dir", "host", "port", "no-open", "json"]));
|
|
217
|
+
if (parsed.positionals.length > 1) {
|
|
218
|
+
throw new UsageError("workbench open accepts at most one source file or directory argument.");
|
|
219
|
+
}
|
|
220
|
+
const workspace = resolveSourceDir(parsed);
|
|
221
|
+
const host = readOptionalStringFlag(parsed.flags.host, "host") ?? "127.0.0.1";
|
|
222
|
+
const port = parsePortFlag(parsed.flags.port);
|
|
223
|
+
const server = await startLocalWorkbenchDevServer({
|
|
224
|
+
workspace,
|
|
225
|
+
host,
|
|
226
|
+
port,
|
|
227
|
+
});
|
|
228
|
+
const result = {
|
|
229
|
+
ok: true,
|
|
230
|
+
url: server.url,
|
|
231
|
+
workspaceRoot: path.resolve(workspace),
|
|
232
|
+
note: LOCAL_DEV_OPEN_LIFECYCLE_NOTE,
|
|
233
|
+
};
|
|
234
|
+
writeOutput(result, parsed, io, (value) => `Workbench open: ${value.url}\nWorkspace: ${value.workspaceRoot}\n${value.note}`);
|
|
235
|
+
if (parsed.flags["no-open"] !== true) {
|
|
236
|
+
await openBrowser(server.url).catch(() => undefined);
|
|
237
|
+
}
|
|
238
|
+
await waitForDevOpenShutdown(server);
|
|
239
|
+
return 0;
|
|
240
|
+
}
|
|
241
|
+
async function waitForDevOpenShutdown(server) {
|
|
242
|
+
await new Promise((resolve, reject) => {
|
|
243
|
+
let closing = false;
|
|
244
|
+
const close = () => {
|
|
245
|
+
if (closing) {
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
closing = true;
|
|
249
|
+
process.off("SIGINT", close);
|
|
250
|
+
process.off("SIGTERM", close);
|
|
251
|
+
server.close().then(resolve, reject);
|
|
252
|
+
};
|
|
253
|
+
process.once("SIGINT", close);
|
|
254
|
+
process.once("SIGTERM", close);
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
async function localInit(argv, io) {
|
|
258
|
+
const parsed = parseArgs(argv);
|
|
259
|
+
rejectUnknownFlags(parsed, new Set([
|
|
260
|
+
"skill",
|
|
261
|
+
"command",
|
|
262
|
+
"agent",
|
|
263
|
+
"from",
|
|
264
|
+
"example",
|
|
265
|
+
"dir",
|
|
266
|
+
"json",
|
|
267
|
+
]));
|
|
268
|
+
const { kind, name } = readInitSelection(parsed);
|
|
269
|
+
const agent = readInitAgent(parsed, kind);
|
|
270
|
+
const workspace = resolveDir(parsed, parsed.positionals[0]);
|
|
271
|
+
const scaffold = createWorkbenchInitScaffold({
|
|
272
|
+
kind,
|
|
273
|
+
name,
|
|
274
|
+
...(agent ? { agent } : {}),
|
|
275
|
+
example: parsed.flags.example === true,
|
|
276
|
+
});
|
|
277
|
+
await fs.mkdir(workspace, { recursive: true });
|
|
278
|
+
await copyInitSeedIfProvided(parsed, workspace, {
|
|
279
|
+
fileTarget: scaffold.seedFileTarget,
|
|
280
|
+
directoryTarget: scaffold.seedDirectoryTarget,
|
|
281
|
+
});
|
|
282
|
+
for (const file of scaffold.files) {
|
|
283
|
+
await writeFileIfMissing(path.join(workspace, file.path), file.content);
|
|
284
|
+
}
|
|
285
|
+
const specPath = path.join(workspace, WORKBENCH_BENCHMARK_FILE);
|
|
286
|
+
writeOutput({
|
|
287
|
+
ok: true,
|
|
288
|
+
dir: workspace,
|
|
289
|
+
specPath,
|
|
290
|
+
kind: scaffold.kind,
|
|
291
|
+
name: scaffold.name,
|
|
292
|
+
subjectRoot: scaffold.subjectRoot,
|
|
293
|
+
}, parsed, io, () => `Initialized ${scaffold.kind} Workbench source directory at ${workspace}`);
|
|
294
|
+
return 0;
|
|
295
|
+
}
|
|
296
|
+
async function localValidate(argv, io) {
|
|
297
|
+
const parsed = parseArgs(argv);
|
|
298
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
299
|
+
const sourceArg = resolveSourceDir(parsed);
|
|
300
|
+
const validation = await readLocalProjectSource(sourceArg)
|
|
301
|
+
.then((projectSource) => ({
|
|
302
|
+
ok: true,
|
|
303
|
+
errors: [],
|
|
304
|
+
warnings: [],
|
|
305
|
+
dir: projectSource.dir,
|
|
306
|
+
specPath: projectSource.specPath,
|
|
307
|
+
plan: buildWorkbenchCheckPlan(projectSource),
|
|
308
|
+
}))
|
|
309
|
+
.catch((error) => ({
|
|
310
|
+
ok: false,
|
|
311
|
+
errors: splitWorkspaceError(error),
|
|
312
|
+
warnings: [],
|
|
313
|
+
}));
|
|
314
|
+
const output = validation;
|
|
315
|
+
writeOutput(output, parsed, io, (record) => {
|
|
316
|
+
const result = record;
|
|
317
|
+
if (!result.ok) {
|
|
318
|
+
return `Spec is invalid:\n${result.errors.map((entry) => `- ${entry}`).join("\n")}`;
|
|
319
|
+
}
|
|
320
|
+
const warningSuffix = result.warnings.length
|
|
321
|
+
? ` with ${result.warnings.length} warning(s)`
|
|
322
|
+
: "";
|
|
323
|
+
return result.plan
|
|
324
|
+
? formatWorkbenchCheckPlan(result.plan, warningSuffix)
|
|
325
|
+
: `Spec is valid${warningSuffix}.`;
|
|
326
|
+
});
|
|
327
|
+
return validation.ok ? 0 : 1;
|
|
328
|
+
}
|
|
329
|
+
function buildWorkbenchCheckPlan(source) {
|
|
330
|
+
return {
|
|
331
|
+
benchmarkName: source.spec.name,
|
|
332
|
+
benchmarkDescription: source.spec.description,
|
|
333
|
+
source: {
|
|
334
|
+
files: sourceFileCount(source),
|
|
335
|
+
yaml: [
|
|
336
|
+
path.relative(source.dir, source.benchmarkPath) || "benchmark.yaml",
|
|
337
|
+
path.relative(source.dir, source.subjectSpecPath) || "subject YAML",
|
|
338
|
+
...(source.optimizerSource !== undefined
|
|
339
|
+
? [path.relative(source.dir, source.optimizerPath ?? "") || "optimizer YAML"]
|
|
340
|
+
: []),
|
|
341
|
+
],
|
|
342
|
+
dockerfile: source.dockerfilePath,
|
|
343
|
+
},
|
|
344
|
+
subject: {
|
|
345
|
+
filesPath: source.spec.subject.files.path,
|
|
346
|
+
files: source.subjectFiles.length,
|
|
347
|
+
},
|
|
348
|
+
optimizer: source.spec.optimizer
|
|
349
|
+
? {
|
|
350
|
+
edits: [...source.spec.optimizer.edits],
|
|
351
|
+
}
|
|
352
|
+
: null,
|
|
353
|
+
engine: {
|
|
354
|
+
resolver: adapterSummary(source.engineResolve),
|
|
355
|
+
path: source.engineResolveFingerprintPath,
|
|
356
|
+
cases: source.caseIds.length,
|
|
357
|
+
files: source.engineResolveFiles.length,
|
|
358
|
+
},
|
|
359
|
+
environment: {
|
|
360
|
+
dockerfile: source.dockerfilePath,
|
|
361
|
+
network: runtimeNetworkSummary(source.spec.environment.network),
|
|
362
|
+
resources: runtimeResourceSummary(source.spec.environment.resources),
|
|
363
|
+
},
|
|
364
|
+
adapters: {
|
|
365
|
+
improve: source.spec.improve ? adapterSummary(source.spec.improve) : null,
|
|
366
|
+
run: adapterSummary(source.spec.run),
|
|
367
|
+
engine: adapterSummary(source.spec.engineRun),
|
|
368
|
+
sources: source.adapters.map(adapterSourceSummary),
|
|
369
|
+
},
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
function formatWorkbenchCheckPlan(plan, warningSuffix) {
|
|
373
|
+
const edits = plan.optimizer?.edits.length
|
|
374
|
+
? plan.optimizer.edits.join(", ")
|
|
375
|
+
: "-";
|
|
376
|
+
const network = plan.environment.network.egress === "allowlist"
|
|
377
|
+
? `allowlist (${plan.environment.network.allow?.join(", ") ?? ""})`
|
|
378
|
+
: plan.environment.network.egress;
|
|
379
|
+
const resources = plan.environment.resources;
|
|
380
|
+
return [
|
|
381
|
+
`Spec is valid${warningSuffix}.`,
|
|
382
|
+
`Benchmark: ${plan.benchmarkName}`,
|
|
383
|
+
`Description: ${plan.benchmarkDescription}`,
|
|
384
|
+
`Source: ${plan.source.files} file(s) (${plan.source.yaml.join(", ")}, ${plan.source.dockerfile})`,
|
|
385
|
+
`Subject files: ${plan.subject.filesPath} (${plan.subject.files} file(s))`,
|
|
386
|
+
`Optimizer edits: ${edits}`,
|
|
387
|
+
`Engine cases: ${plan.engine.cases} case(s) from ${formatAdapterSummary(plan.engine.resolver)} at ${plan.engine.path} (${plan.engine.files} file(s))`,
|
|
388
|
+
`Environment: ${plan.environment.dockerfile}, network ${network}, ${resources.cpu} CPU, ${resources.memoryGb}GB RAM, ${resources.timeoutMinutes}m timeout`,
|
|
389
|
+
`Execution: improve ${plan.adapters.improve ? formatAdapterSummary(plan.adapters.improve) : "not configured"}, subject ${formatAdapterSummary(plan.adapters.run)}, engine ${formatAdapterSummary(plan.adapters.engine)}`,
|
|
390
|
+
...adapterSourceLines(plan.adapters.sources),
|
|
391
|
+
].join("\n");
|
|
392
|
+
}
|
|
393
|
+
function adapterSummary(adapter) {
|
|
394
|
+
const config = readRecord(adapter.with) ?? {};
|
|
395
|
+
const summary = { use: adapter.use };
|
|
396
|
+
if (typeof config.model === "string" && config.model) {
|
|
397
|
+
summary.model = config.model;
|
|
398
|
+
}
|
|
399
|
+
if (typeof config.command === "string" && config.command) {
|
|
400
|
+
summary.command = config.command;
|
|
401
|
+
}
|
|
402
|
+
const judge = readRecord(config.judge);
|
|
403
|
+
if (typeof judge?.use === "string" && judge.use) {
|
|
404
|
+
summary.judge = judge.use;
|
|
405
|
+
}
|
|
406
|
+
if (Array.isArray(config.criteria)) {
|
|
407
|
+
summary.criteria = config.criteria.length;
|
|
408
|
+
}
|
|
409
|
+
return summary;
|
|
410
|
+
}
|
|
411
|
+
function formatAdapterSummary(summary) {
|
|
412
|
+
const details = [
|
|
413
|
+
summary.model,
|
|
414
|
+
summary.judge ? `judge ${summary.judge}` : "",
|
|
415
|
+
summary.criteria === undefined ? "" : `${summary.criteria} criteria`,
|
|
416
|
+
summary.command ? truncateCommand(summary.command) : "",
|
|
417
|
+
].filter(Boolean);
|
|
418
|
+
return details.length ? `${summary.use} (${details.join(", ")})` : summary.use;
|
|
419
|
+
}
|
|
420
|
+
function adapterSourceSummary(adapter) {
|
|
421
|
+
return {
|
|
422
|
+
id: adapter.manifest.id,
|
|
423
|
+
kind: adapter.kind,
|
|
424
|
+
declaredSource: adapter.declaredSource,
|
|
425
|
+
resolvedSource: adapter.source,
|
|
426
|
+
stability: adapter.stability,
|
|
427
|
+
...(adapter.overridesDefault ? { overridesDefault: true } : {}),
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
function adapterSourceLines(sources) {
|
|
431
|
+
const external = sources.filter((source) => source.kind !== "default");
|
|
432
|
+
if (external.length === 0) {
|
|
433
|
+
return [];
|
|
434
|
+
}
|
|
435
|
+
return [
|
|
436
|
+
`Adapter sources: ${external.map(formatAdapterSourceSummary).join("; ")}`,
|
|
437
|
+
];
|
|
438
|
+
}
|
|
439
|
+
function formatAdapterSourceSummary(source) {
|
|
440
|
+
const override = source.overridesDefault ? " overrides default" : "";
|
|
441
|
+
return `${source.id} ${source.stability}${override} ${formatAdapterResolution(source)}`;
|
|
442
|
+
}
|
|
443
|
+
function formatAdapterResolution(source) {
|
|
444
|
+
const resolution = source.declaredSource === source.resolvedSource
|
|
445
|
+
? source.declaredSource
|
|
446
|
+
: `${source.declaredSource} -> ${source.resolvedSource}`;
|
|
447
|
+
return resolution;
|
|
448
|
+
}
|
|
449
|
+
function truncateCommand(command) {
|
|
450
|
+
return command.length > 80 ? `${command.slice(0, 77)}...` : command;
|
|
451
|
+
}
|
|
452
|
+
function runtimeNetworkSummary(configValue) {
|
|
453
|
+
const network = readRecord(configValue) ?? {};
|
|
454
|
+
const egress = network.egress === "none" || network.egress === "allowlist"
|
|
455
|
+
? network.egress
|
|
456
|
+
: "open";
|
|
457
|
+
if (egress !== "allowlist") {
|
|
458
|
+
return { egress };
|
|
459
|
+
}
|
|
460
|
+
const allow = Array.isArray(network.allow)
|
|
461
|
+
? network.allow.flatMap((entry) => typeof entry === "string" ? [entry] : [])
|
|
462
|
+
: [];
|
|
463
|
+
return {
|
|
464
|
+
egress,
|
|
465
|
+
allow,
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
function runtimeResourceSummary(configValue) {
|
|
469
|
+
const resources = readRecord(configValue) ?? {};
|
|
470
|
+
return {
|
|
471
|
+
cpu: readPositiveNumber(resources.cpu, 2),
|
|
472
|
+
memoryGb: readPositiveNumber(resources.memoryGb, 4),
|
|
473
|
+
diskGb: readPositiveNumber(resources.diskGb, 10),
|
|
474
|
+
timeoutMinutes: readPositiveNumber(resources.timeoutMinutes, 20),
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
function readPositiveNumber(value, fallback) {
|
|
478
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
479
|
+
? value
|
|
480
|
+
: fallback;
|
|
481
|
+
}
|
|
482
|
+
function splitWorkspaceError(error) {
|
|
483
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
484
|
+
return message.split(/\n+/u).map((entry) => entry.trim()).filter(Boolean);
|
|
485
|
+
}
|
|
486
|
+
async function localRun(argv, io, runtimeOptions) {
|
|
487
|
+
const parsed = parseArgs(argv);
|
|
488
|
+
rejectUnknownFlags(parsed, new Set(["dir", "optimizer", "from", "budget", "samples", "json"]));
|
|
489
|
+
const sourceArg = resolveSourceDir(parsed);
|
|
490
|
+
const projectSource = await readLocalProjectSource(sourceArg, {
|
|
491
|
+
optimizerPath: asOptionalString(parsed.flags.optimizer),
|
|
492
|
+
});
|
|
493
|
+
const workspace = projectSource.dir;
|
|
494
|
+
if (!projectSource.spec.optimizer) {
|
|
495
|
+
throw new UsageError("Optimizer YAML is required for workbench improve.");
|
|
496
|
+
}
|
|
497
|
+
const executionProject = await resolveLocalProjectForExecution(workspace, projectSource.specSource);
|
|
498
|
+
const { spec, adapterManifests } = executionProject;
|
|
499
|
+
const budget = parsePositiveInt(parsed.flags.budget, 1, "budget");
|
|
500
|
+
const samples = parsePositiveInt(parsed.flags.samples, 1, "samples");
|
|
501
|
+
const engineResolveFiles = normalizeSurfaceFiles(projectSource.engineResolveFiles);
|
|
502
|
+
const engineCases = projectSource.engineCases;
|
|
503
|
+
const caseIds = engineCases.map((bundle) => bundle.id);
|
|
504
|
+
if (caseIds.length === 0) {
|
|
505
|
+
throw new UsageError("Engine resolver must emit at least one case.");
|
|
506
|
+
}
|
|
507
|
+
requireValidRunEnvelope({
|
|
508
|
+
workflow: "improve",
|
|
509
|
+
budget,
|
|
510
|
+
samples,
|
|
511
|
+
caseCount: caseIds.length,
|
|
512
|
+
});
|
|
513
|
+
const environmentRefs = await ensureLocalDockerfileEnvironments(workspace, spec, engineCases);
|
|
514
|
+
const benchmarkFingerprint = await readLocalBenchmarkFingerprint(workspace);
|
|
515
|
+
const runId = `run_local_${Date.now().toString(36)}`;
|
|
516
|
+
const startedAt = new Date().toISOString();
|
|
517
|
+
let snapshot = await loadLocalArchive(workspace);
|
|
518
|
+
const baseSubject = await ensureLocalImproveBaseSubject({
|
|
519
|
+
parsed,
|
|
520
|
+
sourceArg,
|
|
521
|
+
workspace,
|
|
522
|
+
projectSource,
|
|
523
|
+
samples,
|
|
524
|
+
io,
|
|
525
|
+
runtimeOptions,
|
|
526
|
+
});
|
|
527
|
+
let currentBaseId = baseSubject.id;
|
|
528
|
+
let completedJobCount = 0;
|
|
529
|
+
let failedJobCount = 0;
|
|
530
|
+
const failedJobs = [];
|
|
531
|
+
const events = [
|
|
532
|
+
createLocalEvent("run_started", startedAt, {
|
|
533
|
+
runId,
|
|
534
|
+
detail: { budget, samples, strategy: "greedy" },
|
|
535
|
+
}),
|
|
536
|
+
];
|
|
537
|
+
const devCapacity = await localDevelopmentCapacity(workspace);
|
|
538
|
+
const runTraceJobs = [];
|
|
539
|
+
const attempts = budget;
|
|
540
|
+
for (let attemptIndex = 0; attemptIndex < attempts; attemptIndex += 1) {
|
|
541
|
+
snapshot = await loadLocalArchive(workspace);
|
|
542
|
+
const activeSubject = readLocalSubject(snapshot, currentBaseId);
|
|
543
|
+
const baseFiles = filterSubjectSourceFiles(readLocalSubjectFiles(snapshot, activeSubject.id));
|
|
544
|
+
if (baseFiles.length === 0) {
|
|
545
|
+
throw new UsageError("Subject snapshot must include at least one file.");
|
|
546
|
+
}
|
|
547
|
+
const subjectRevisionTraceFiles = createSubjectRevisionTraceInputFiles({
|
|
548
|
+
runId,
|
|
549
|
+
jobs: runTraceJobs,
|
|
550
|
+
events,
|
|
551
|
+
});
|
|
552
|
+
const subjectId = `subject_${runId.replace(/^run_/u, "")}_${String(attemptIndex + 1).padStart(3, "0")}`;
|
|
553
|
+
const plannedSubjectRevision = planWorkbenchExecutionJobsForPurpose({
|
|
554
|
+
ownerUserId: "local",
|
|
555
|
+
projectId: "local",
|
|
556
|
+
runId,
|
|
557
|
+
subjectId,
|
|
558
|
+
attemptIndex,
|
|
559
|
+
samples,
|
|
560
|
+
caseIds,
|
|
561
|
+
engineCases,
|
|
562
|
+
spec,
|
|
563
|
+
workflow: "improve",
|
|
564
|
+
purpose: "improve",
|
|
565
|
+
now: new Date().toISOString(),
|
|
566
|
+
baseFiles,
|
|
567
|
+
traceFiles: subjectRevisionTraceFiles,
|
|
568
|
+
environmentRef: environmentRefs.defaultRef,
|
|
569
|
+
baseId: activeSubject.id,
|
|
570
|
+
})[0];
|
|
571
|
+
const subjectRevisionJobs = await executeLocalDevelopmentDag({
|
|
572
|
+
jobs: [plannedSubjectRevision],
|
|
573
|
+
spec,
|
|
574
|
+
adapterManifests,
|
|
575
|
+
baseFiles,
|
|
576
|
+
engineResolveFiles,
|
|
577
|
+
engineCases,
|
|
578
|
+
traceFiles: subjectRevisionTraceFiles,
|
|
579
|
+
capacity: devCapacity,
|
|
580
|
+
});
|
|
581
|
+
const subjectRevision = subjectRevisionJobs[0];
|
|
582
|
+
const completedJobs = [subjectRevision];
|
|
583
|
+
if (subjectRevision.status === "succeeded") {
|
|
584
|
+
const subjectRevisionFiles = completedJobOutputFiles(subjectRevision).length > 0
|
|
585
|
+
? normalizeSurfaceFiles(completedJobOutputFiles(subjectRevision).filter((file) => !file.path.startsWith(".workbench/")))
|
|
586
|
+
: baseFiles;
|
|
587
|
+
const attemptJobs = planWorkbenchExecutionJobsForPurpose({
|
|
588
|
+
ownerUserId: "local",
|
|
589
|
+
projectId: "local",
|
|
590
|
+
runId,
|
|
591
|
+
subjectId,
|
|
592
|
+
attemptIndex,
|
|
593
|
+
samples,
|
|
594
|
+
now: new Date().toISOString(),
|
|
595
|
+
caseIds,
|
|
596
|
+
engineCases,
|
|
597
|
+
spec,
|
|
598
|
+
environmentRefsByCase: environmentRefs.byCase,
|
|
599
|
+
workflow: "improve",
|
|
600
|
+
purpose: "attempt",
|
|
601
|
+
});
|
|
602
|
+
const dagJobs = await executeLocalDevelopmentDag({
|
|
603
|
+
jobs: [subjectRevision, ...attemptJobs],
|
|
604
|
+
spec,
|
|
605
|
+
adapterManifests,
|
|
606
|
+
baseFiles: subjectRevisionFiles,
|
|
607
|
+
engineResolveFiles,
|
|
608
|
+
engineCases,
|
|
609
|
+
capacity: devCapacity,
|
|
610
|
+
});
|
|
611
|
+
completedJobs.splice(0, completedJobs.length, ...dagJobs);
|
|
612
|
+
}
|
|
613
|
+
runTraceJobs.push(...completedJobs);
|
|
614
|
+
const materialized = materializeWorkbenchRunResult({
|
|
615
|
+
runId,
|
|
616
|
+
benchmarkFingerprint,
|
|
617
|
+
sourceYaml: projectSource.specSource,
|
|
618
|
+
benchmarkSourceFiles: authoredBenchmarkSourceFiles(projectSource),
|
|
619
|
+
startedAt,
|
|
620
|
+
spec,
|
|
621
|
+
jobs: completedJobs,
|
|
622
|
+
previousSubject: activeSubject,
|
|
623
|
+
existingSubjectCount: snapshot.subjects.length,
|
|
624
|
+
});
|
|
625
|
+
for (const subject of materialized.subjects) {
|
|
626
|
+
snapshot = upsertLocalSubject(snapshot, subject, materialized.subjectFiles[subject.id] ?? []);
|
|
627
|
+
events.push(createLocalEvent("subject_created", subject.createdAt, {
|
|
628
|
+
runId,
|
|
629
|
+
subjectId: subject.id,
|
|
630
|
+
baseId: subject.baseId,
|
|
631
|
+
status: subject.status,
|
|
632
|
+
metrics: subject.metrics,
|
|
633
|
+
}));
|
|
634
|
+
}
|
|
635
|
+
for (const evaluation of materialized.evaluations) {
|
|
636
|
+
snapshot = upsertLocalEvaluation(snapshot, evaluation);
|
|
637
|
+
}
|
|
638
|
+
snapshot = setLocalActive(snapshot, materialized.activeSubjectId);
|
|
639
|
+
currentBaseId = materialized.activeSubjectId ?? currentBaseId;
|
|
640
|
+
completedJobCount += materialized.completedJobCount;
|
|
641
|
+
failedJobCount += materialized.failedJobCount;
|
|
642
|
+
failedJobs.push(...completedJobs
|
|
643
|
+
.filter((job) => job.status === "failed")
|
|
644
|
+
.map((job) => ({
|
|
645
|
+
id: job.id,
|
|
646
|
+
purpose: workbenchExecutionPurpose(job),
|
|
647
|
+
error: job.error ?? "Job failed without an error message.",
|
|
648
|
+
})));
|
|
649
|
+
events.push(createLocalEvent("active_changed", new Date().toISOString(), {
|
|
650
|
+
runId,
|
|
651
|
+
subjectId: materialized.activeSubjectId ?? undefined,
|
|
652
|
+
activeId: materialized.activeSubjectId ?? undefined,
|
|
653
|
+
status: materialized.selectedSubject?.status,
|
|
654
|
+
metrics: materialized.selectedSubject?.metrics,
|
|
655
|
+
}));
|
|
656
|
+
await saveLocalJobs(workspace, completedJobs);
|
|
657
|
+
await saveLocalArchive(workspace, snapshot);
|
|
658
|
+
}
|
|
659
|
+
snapshot = await loadLocalArchive(workspace);
|
|
660
|
+
const finishedAt = new Date().toISOString();
|
|
661
|
+
const run = {
|
|
662
|
+
id: runId,
|
|
663
|
+
workflow: "improve",
|
|
664
|
+
benchmarkFingerprint,
|
|
665
|
+
status: "finished",
|
|
666
|
+
startedAt,
|
|
667
|
+
finishedAt,
|
|
668
|
+
durationMs: Math.max(0, Date.parse(finishedAt) - Date.parse(startedAt)),
|
|
669
|
+
optimizer: formatSpecOptimizer(spec),
|
|
670
|
+
engineRun: spec.engineRun.use,
|
|
671
|
+
strategy: "greedy",
|
|
672
|
+
budget,
|
|
673
|
+
repairBudget: 0,
|
|
674
|
+
attemptsRequested: budget,
|
|
675
|
+
attemptsExecuted: budget,
|
|
676
|
+
samples,
|
|
677
|
+
stoppedReason: "budget_exhausted",
|
|
678
|
+
outcome: failedJobCount > 0 ? "error" : "ok",
|
|
679
|
+
};
|
|
680
|
+
events.push(createLocalEvent("run_finished", finishedAt, {
|
|
681
|
+
runId,
|
|
682
|
+
detail: {
|
|
683
|
+
outcome: run.outcome ?? null,
|
|
684
|
+
attemptsExecuted: run.attemptsExecuted,
|
|
685
|
+
durationMs: run.durationMs ?? null,
|
|
686
|
+
},
|
|
687
|
+
}));
|
|
688
|
+
snapshot = appendLocalRun(snapshot, run, events);
|
|
689
|
+
await saveLocalArchive(workspace, snapshot);
|
|
690
|
+
const selected = snapshot.activeId
|
|
691
|
+
? readLocalSubject(snapshot, snapshot.activeId)
|
|
692
|
+
: null;
|
|
693
|
+
const result = {
|
|
694
|
+
ok: failedJobCount === 0,
|
|
695
|
+
runId,
|
|
696
|
+
activeSubjectId: snapshot.activeId,
|
|
697
|
+
selectedSubject: selected,
|
|
698
|
+
completedJobCount,
|
|
699
|
+
failedJobCount,
|
|
700
|
+
failedJobs,
|
|
701
|
+
localView: localDevViewHint(workspace),
|
|
702
|
+
};
|
|
703
|
+
writeOutput(result, parsed, io, () => {
|
|
704
|
+
const metricValue = selected?.metrics?.score ?? "n/a";
|
|
705
|
+
const firstFailure = result.failedJobs[0];
|
|
706
|
+
const failureDetail = firstFailure
|
|
707
|
+
? `\nFirst failed job ${firstFailure.id}${firstFailure.purpose ? ` (${firstFailure.purpose})` : ""}: ${firstFailure.error}`
|
|
708
|
+
: "";
|
|
709
|
+
const viewDetail = failedJobCount === 0
|
|
710
|
+
? `\nOpen local view: ${result.localView.command}\n${result.localView.note}`
|
|
711
|
+
: "";
|
|
712
|
+
return `Run ${runId} finished. Active subject: ${snapshot.activeId ?? "none"} (score: ${metricValue}).${failureDetail}${viewDetail}`;
|
|
713
|
+
});
|
|
714
|
+
return failedJobCount === 0 ? 0 : 1;
|
|
715
|
+
}
|
|
716
|
+
async function ensureLocalImproveBaseSubject(args) {
|
|
717
|
+
let snapshot = await loadLocalArchive(args.workspace);
|
|
718
|
+
const explicitBase = asOptionalString(args.parsed.flags.from);
|
|
719
|
+
const benchmarkFingerprint = await readLocalBenchmarkFingerprint(args.workspace);
|
|
720
|
+
if (explicitBase) {
|
|
721
|
+
let subject = readLocalSubject(snapshot, explicitBase);
|
|
722
|
+
if (subject.benchmarkFingerprint !== benchmarkFingerprint) {
|
|
723
|
+
throw new UsageError(`Base subject ${explicitBase} belongs to benchmark ${subject.benchmarkFingerprint}, not ${benchmarkFingerprint}.`);
|
|
724
|
+
}
|
|
725
|
+
if (!subject.subjectFingerprint) {
|
|
726
|
+
throw new UsageError(`Base subject ${explicitBase} is missing a subject fingerprint.`);
|
|
727
|
+
}
|
|
728
|
+
if (subject.status !== "evaluated" && !subject.eval) {
|
|
729
|
+
const code = await localEvaluateSubject(["--dir", args.workspace, "--subject", explicitBase, "--samples", String(args.samples), "--json"], createSilentIo(args.io), args.runtimeOptions);
|
|
730
|
+
if (code !== 0) {
|
|
731
|
+
throw new UsageError(`Base subject ${explicitBase} eval failed; improve was not started.`);
|
|
732
|
+
}
|
|
733
|
+
snapshot = await loadLocalArchive(args.workspace);
|
|
734
|
+
subject = readLocalSubject(snapshot, explicitBase);
|
|
735
|
+
}
|
|
736
|
+
return subject;
|
|
737
|
+
}
|
|
738
|
+
const subjectFingerprint = localSubjectFingerprint(args.projectSource);
|
|
739
|
+
const existing = snapshot.subjects.find((subject) => subject.benchmarkFingerprint === benchmarkFingerprint &&
|
|
740
|
+
subject.subjectFingerprint === subjectFingerprint &&
|
|
741
|
+
(subject.status === "evaluated" || Boolean(subject.eval)));
|
|
742
|
+
if (existing) {
|
|
743
|
+
return existing;
|
|
744
|
+
}
|
|
745
|
+
const evalArgs = args.parsed.positionals.length > 0
|
|
746
|
+
? [args.sourceArg, "--samples", String(args.samples), "--json"]
|
|
747
|
+
: ["--dir", args.workspace, "--samples", String(args.samples), "--json"];
|
|
748
|
+
const code = await localEvaluateSubject(evalArgs, createSilentIo(args.io), args.runtimeOptions);
|
|
749
|
+
if (code !== 0) {
|
|
750
|
+
throw new UsageError("Parent subject eval failed; improve was not started.");
|
|
751
|
+
}
|
|
752
|
+
snapshot = await loadLocalArchive(args.workspace);
|
|
753
|
+
const evaluated = snapshot.subjects.find((subject) => subject.benchmarkFingerprint === benchmarkFingerprint &&
|
|
754
|
+
subject.subjectFingerprint === subjectFingerprint &&
|
|
755
|
+
(subject.status === "evaluated" || Boolean(subject.eval)));
|
|
756
|
+
if (!evaluated) {
|
|
757
|
+
throw new UsageError("Parent subject eval did not produce an evaluated subject.");
|
|
758
|
+
}
|
|
759
|
+
return evaluated;
|
|
760
|
+
}
|
|
761
|
+
function createSilentIo(io) {
|
|
762
|
+
const sink = new class extends Writable {
|
|
763
|
+
_write(_chunk, _encoding, callback) {
|
|
764
|
+
callback();
|
|
765
|
+
}
|
|
766
|
+
}();
|
|
767
|
+
return {
|
|
768
|
+
stdin: io.stdin,
|
|
769
|
+
stdout: sink,
|
|
770
|
+
stderr: io.stderr,
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
async function localEvaluateSubject(argv, io, runtimeOptions) {
|
|
774
|
+
void runtimeOptions;
|
|
775
|
+
const parsed = parseArgs(argv);
|
|
776
|
+
rejectUnknownFlags(parsed, new Set(["dir", "subject", "samples", "json"]));
|
|
777
|
+
const sourceArg = resolveSourceDir(parsed);
|
|
778
|
+
const projectSource = await readLocalProjectSource(sourceArg);
|
|
779
|
+
const workspace = projectSource.dir;
|
|
780
|
+
const executionProject = await resolveLocalProjectForExecution(workspace, projectSource.specSource);
|
|
781
|
+
const { spec, adapterManifests } = executionProject;
|
|
782
|
+
const samples = parsePositiveInt(parsed.flags.samples, 1, "samples");
|
|
783
|
+
const engineResolveFiles = normalizeSurfaceFiles(projectSource.engineResolveFiles);
|
|
784
|
+
const engineCases = projectSource.engineCases;
|
|
785
|
+
const caseIds = engineCases.map((bundle) => bundle.id);
|
|
786
|
+
if (caseIds.length === 0) {
|
|
787
|
+
throw new UsageError("Engine resolver must emit at least one case.");
|
|
788
|
+
}
|
|
789
|
+
requireValidRunEnvelope({
|
|
790
|
+
workflow: "eval",
|
|
791
|
+
budget: 1,
|
|
792
|
+
samples,
|
|
793
|
+
caseCount: caseIds.length,
|
|
794
|
+
});
|
|
795
|
+
const environmentRefs = await ensureLocalDockerfileEnvironments(workspace, spec, engineCases);
|
|
796
|
+
let snapshot = await loadLocalArchive(workspace);
|
|
797
|
+
const benchmarkFingerprint = await readLocalBenchmarkFingerprint(workspace);
|
|
798
|
+
const sourceSubjectFingerprint = localSubjectFingerprint(projectSource);
|
|
799
|
+
const explicitSubjectId = asOptionalString(parsed.flags.subject);
|
|
800
|
+
const existingSourceSubject = snapshot.subjects.find((subject) => subject.benchmarkFingerprint === benchmarkFingerprint &&
|
|
801
|
+
subject.subjectFingerprint === sourceSubjectFingerprint);
|
|
802
|
+
const subjectId = explicitSubjectId ?? existingSourceSubject?.id ?? `subject_${sourceSubjectFingerprint.slice(0, 12)}`;
|
|
803
|
+
const existingSubject = snapshot.subjects.find((subject) => subject.id === subjectId);
|
|
804
|
+
const files = filterSubjectSourceFiles(existingSubject
|
|
805
|
+
? readLocalSubjectFiles(snapshot, subjectId)
|
|
806
|
+
: normalizeSurfaceFiles(projectSource.subjectFiles));
|
|
807
|
+
const runId = `eval_local_${Date.now().toString(36)}`;
|
|
808
|
+
const evaluatedSubjectId = subjectId;
|
|
809
|
+
const startedAt = new Date().toISOString();
|
|
810
|
+
const baseline = createRuntimeBaselineSubjectJob({
|
|
811
|
+
ownerUserId: "local",
|
|
812
|
+
projectId: "local",
|
|
813
|
+
runId,
|
|
814
|
+
subjectId: evaluatedSubjectId,
|
|
815
|
+
attemptIndex: 0,
|
|
816
|
+
files,
|
|
817
|
+
now: startedAt,
|
|
818
|
+
baseId: null,
|
|
819
|
+
});
|
|
820
|
+
const completedJobs = [baseline];
|
|
821
|
+
const attemptJobs = planWorkbenchExecutionJobsForPurpose({
|
|
822
|
+
ownerUserId: "local",
|
|
823
|
+
projectId: "local",
|
|
824
|
+
runId,
|
|
825
|
+
subjectId: evaluatedSubjectId,
|
|
826
|
+
attemptIndex: 0,
|
|
827
|
+
samples,
|
|
828
|
+
now: startedAt,
|
|
829
|
+
caseIds,
|
|
830
|
+
engineCases,
|
|
831
|
+
spec,
|
|
832
|
+
environmentRefsByCase: environmentRefs.byCase,
|
|
833
|
+
workflow: "eval",
|
|
834
|
+
purpose: "attempt",
|
|
835
|
+
});
|
|
836
|
+
const dagJobs = await executeLocalDevelopmentDag({
|
|
837
|
+
jobs: [baseline, ...attemptJobs],
|
|
838
|
+
spec,
|
|
839
|
+
adapterManifests,
|
|
840
|
+
baseFiles: files,
|
|
841
|
+
engineResolveFiles,
|
|
842
|
+
engineCases,
|
|
843
|
+
capacity: await localDevelopmentCapacity(workspace),
|
|
844
|
+
});
|
|
845
|
+
completedJobs.splice(0, completedJobs.length, ...dagJobs);
|
|
846
|
+
const materialized = materializeWorkbenchRunResult({
|
|
847
|
+
runId,
|
|
848
|
+
benchmarkFingerprint,
|
|
849
|
+
sourceYaml: projectSource.specSource,
|
|
850
|
+
benchmarkSourceFiles: authoredBenchmarkSourceFiles(projectSource),
|
|
851
|
+
subjectFingerprint: existingSubject?.subjectFingerprint ?? sourceSubjectFingerprint,
|
|
852
|
+
...(!existingSubject || existingSubject.subjectFingerprint === sourceSubjectFingerprint
|
|
853
|
+
? { subjectSourceFiles: authoredSubjectSourceFiles(projectSource) }
|
|
854
|
+
: {}),
|
|
855
|
+
startedAt,
|
|
856
|
+
spec,
|
|
857
|
+
jobs: completedJobs,
|
|
858
|
+
previousSubject: null,
|
|
859
|
+
existingSubjectCount: snapshot.subjects.length,
|
|
860
|
+
});
|
|
861
|
+
for (const subjectRecord of materialized.subjects) {
|
|
862
|
+
snapshot = upsertLocalSubject(snapshot, subjectRecord, materialized.subjectFiles[subjectRecord.id] ?? []);
|
|
863
|
+
}
|
|
864
|
+
if (materialized.activeSubjectId) {
|
|
865
|
+
snapshot = setLocalActive(snapshot, materialized.activeSubjectId);
|
|
866
|
+
}
|
|
867
|
+
for (const evaluation of materialized.evaluations) {
|
|
868
|
+
snapshot = upsertLocalEvaluation(snapshot, evaluation);
|
|
869
|
+
}
|
|
870
|
+
const finishedAt = new Date().toISOString();
|
|
871
|
+
snapshot = appendLocalRun(snapshot, {
|
|
872
|
+
id: runId,
|
|
873
|
+
workflow: "eval",
|
|
874
|
+
benchmarkFingerprint,
|
|
875
|
+
status: "finished",
|
|
876
|
+
startedAt,
|
|
877
|
+
finishedAt,
|
|
878
|
+
durationMs: Math.max(0, Date.parse(finishedAt) - Date.parse(startedAt)),
|
|
879
|
+
optimizer: "none",
|
|
880
|
+
engineRun: spec.engineRun.use,
|
|
881
|
+
strategy: "direct",
|
|
882
|
+
budget: 1,
|
|
883
|
+
repairBudget: 0,
|
|
884
|
+
attemptsRequested: 1,
|
|
885
|
+
attemptsExecuted: 1,
|
|
886
|
+
samples,
|
|
887
|
+
stoppedReason: "completed",
|
|
888
|
+
outcome: materialized.failedJobCount > 0 ? "error" : "ok",
|
|
889
|
+
}, []);
|
|
890
|
+
await saveLocalJobs(workspace, completedJobs);
|
|
891
|
+
await saveLocalArchive(workspace, snapshot);
|
|
892
|
+
const evaluation = materialized.evaluations[0] ?? null;
|
|
893
|
+
const result = {
|
|
894
|
+
ok: materialized.failedJobCount === 0,
|
|
895
|
+
evaluation,
|
|
896
|
+
resultId: evaluation?.id ?? null,
|
|
897
|
+
subjectId: evaluatedSubjectId,
|
|
898
|
+
completedJobCount: materialized.completedJobCount,
|
|
899
|
+
failedJobCount: materialized.failedJobCount,
|
|
900
|
+
localView: localDevViewHint(workspace),
|
|
901
|
+
};
|
|
902
|
+
writeOutput(result, parsed, io, ({ resultId, subjectId: evaluatedSubjectId }) => `Evaluation ${resultId ?? runId} finished for ${evaluatedSubjectId}.\nOpen local view: ${result.localView.command}\n${result.localView.note}`);
|
|
903
|
+
return materialized.failedJobCount === 0 ? 0 : 1;
|
|
904
|
+
}
|
|
905
|
+
function localDevViewHint(workspace) {
|
|
906
|
+
return {
|
|
907
|
+
command: `workbench open --dir ${shellQuote(path.resolve(workspace))}`,
|
|
908
|
+
note: LOCAL_DEV_OPEN_LIFECYCLE_NOTE,
|
|
909
|
+
};
|
|
910
|
+
}
|
|
911
|
+
async function readLocalBenchmarkFingerprint(workspace) {
|
|
912
|
+
return localBenchmarkFingerprint(await readLocalProjectSource(workspace));
|
|
913
|
+
}
|
|
914
|
+
function authoredSubjectSourceFiles(projectSource) {
|
|
915
|
+
return [{
|
|
916
|
+
path: path.relative(projectSource.dir, projectSource.subjectSpecPath).split(path.sep).join("/"),
|
|
917
|
+
kind: "text",
|
|
918
|
+
encoding: "utf8",
|
|
919
|
+
content: projectSource.subjectSource,
|
|
920
|
+
executable: false,
|
|
921
|
+
}];
|
|
922
|
+
}
|
|
923
|
+
function authoredBenchmarkSourceFiles(projectSource) {
|
|
924
|
+
return [{
|
|
925
|
+
path: path.relative(projectSource.dir, projectSource.benchmarkPath).split(path.sep).join("/"),
|
|
926
|
+
kind: "text",
|
|
927
|
+
encoding: "utf8",
|
|
928
|
+
content: projectSource.benchmarkSource,
|
|
929
|
+
executable: false,
|
|
930
|
+
}];
|
|
931
|
+
}
|
|
932
|
+
function checkpointSubjectFingerprint(files) {
|
|
933
|
+
const hash = createHash("sha256");
|
|
934
|
+
hash.update("workbench-checkpoint-subject-v1\0");
|
|
935
|
+
hashSurfaceFiles(hash, files);
|
|
936
|
+
return hash.digest("hex");
|
|
937
|
+
}
|
|
938
|
+
function hashSurfaceFiles(hash, files) {
|
|
939
|
+
for (const file of files.slice().sort((left, right) => left.path.localeCompare(right.path))) {
|
|
940
|
+
hash.update("\0file\0");
|
|
941
|
+
hash.update(file.path);
|
|
942
|
+
hash.update("\0");
|
|
943
|
+
hash.update(file.encoding ?? "utf8");
|
|
944
|
+
hash.update("\0");
|
|
945
|
+
hash.update(file.content);
|
|
946
|
+
hash.update("\0");
|
|
947
|
+
hash.update(file.executable ? "1" : "0");
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
function shellQuote(value) {
|
|
951
|
+
return `'${value.replace(/'/gu, "'\\''")}'`;
|
|
952
|
+
}
|
|
953
|
+
function resolveProjectPath(root, filePath) {
|
|
954
|
+
return path.isAbsolute(filePath) ? filePath : path.join(root, filePath);
|
|
955
|
+
}
|
|
956
|
+
async function executeLocalDevelopmentJob(args) {
|
|
957
|
+
return await executeWorkbenchExecutionJob(args, {
|
|
958
|
+
sandboxProvider: DOCKER_SANDBOX_BACKEND,
|
|
959
|
+
loadLocalAdapterAuthProfiles: true,
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
async function executeLocalDevelopmentDag(args) {
|
|
963
|
+
const completedById = new Map(args.jobs
|
|
964
|
+
.filter(isTerminalLocalJob)
|
|
965
|
+
.map((job) => [job.id, job]));
|
|
966
|
+
const result = await runWorkbenchExecutionDag({
|
|
967
|
+
jobs: args.jobs,
|
|
968
|
+
capacity: args.capacity,
|
|
969
|
+
sandboxProvider: DOCKER_SANDBOX_BACKEND,
|
|
970
|
+
executeJob: async (job) => {
|
|
971
|
+
return await executeLocalDevelopmentJob({
|
|
972
|
+
job,
|
|
973
|
+
spec: args.spec,
|
|
974
|
+
adapterManifests: args.adapterManifests,
|
|
975
|
+
baseFiles: args.baseFiles,
|
|
976
|
+
engineResolveFiles: args.engineResolveFiles,
|
|
977
|
+
engineCases: args.engineCases,
|
|
978
|
+
...(args.traceFiles ? { traceFiles: args.traceFiles } : {}),
|
|
979
|
+
});
|
|
980
|
+
},
|
|
981
|
+
onJobFinished: (job) => {
|
|
982
|
+
completedById.set(job.id, job);
|
|
983
|
+
},
|
|
984
|
+
});
|
|
985
|
+
return result.jobs;
|
|
986
|
+
}
|
|
987
|
+
async function localDevelopmentCapacity(workspace) {
|
|
988
|
+
const envCapacity = localDevelopmentCapacityFromEnv();
|
|
989
|
+
if (envCapacity) {
|
|
990
|
+
return envCapacity;
|
|
991
|
+
}
|
|
992
|
+
const filesystem = await fs.statfs(workspace);
|
|
993
|
+
const availableDiskGb = (filesystem.bavail * filesystem.bsize) / (1024 ** 3);
|
|
994
|
+
return {
|
|
995
|
+
cpu: Math.max(1, typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length),
|
|
996
|
+
memoryGb: Math.max(1, os.totalmem() / (1024 ** 3)),
|
|
997
|
+
diskGb: Math.max(1, availableDiskGb),
|
|
998
|
+
};
|
|
999
|
+
}
|
|
1000
|
+
function localDevelopmentCapacityFromEnv() {
|
|
1001
|
+
const names = [
|
|
1002
|
+
"WORKBENCH_HOST_CPU",
|
|
1003
|
+
"WORKBENCH_HOST_MEMORY_GB",
|
|
1004
|
+
"WORKBENCH_HOST_DISK_GB",
|
|
1005
|
+
];
|
|
1006
|
+
const values = names.map((name) => process.env[name]);
|
|
1007
|
+
if (values.every((value) => value === undefined || value === "")) {
|
|
1008
|
+
return null;
|
|
1009
|
+
}
|
|
1010
|
+
if (values.some((value) => value === undefined || value === "")) {
|
|
1011
|
+
throw new UsageError(`${names.join(", ")} must be set together for local dev capacity.`);
|
|
1012
|
+
}
|
|
1013
|
+
return {
|
|
1014
|
+
cpu: readPositiveEnvNumber(names[0]),
|
|
1015
|
+
memoryGb: readPositiveEnvNumber(names[1]),
|
|
1016
|
+
diskGb: readPositiveEnvNumber(names[2]),
|
|
1017
|
+
};
|
|
1018
|
+
}
|
|
1019
|
+
function readPositiveEnvNumber(name) {
|
|
1020
|
+
const value = Number(process.env[name]);
|
|
1021
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
1022
|
+
throw new UsageError(`${name} must be a positive number.`);
|
|
1023
|
+
}
|
|
1024
|
+
return value;
|
|
1025
|
+
}
|
|
1026
|
+
function isTerminalLocalJob(job) {
|
|
1027
|
+
return job.status === "succeeded" || job.status === "failed" || job.status === "cancelled";
|
|
1028
|
+
}
|
|
1029
|
+
async function ensureLocalDockerfileEnvironments(workspace, spec, engineCases) {
|
|
1030
|
+
const cache = new Map();
|
|
1031
|
+
const ensure = (runtime) => {
|
|
1032
|
+
const key = runtime.dockerfile;
|
|
1033
|
+
const existing = cache.get(key);
|
|
1034
|
+
if (existing) {
|
|
1035
|
+
return existing;
|
|
1036
|
+
}
|
|
1037
|
+
const pending = ensureLocalDockerfileEnvironment(workspace, spec, runtime);
|
|
1038
|
+
cache.set(key, pending);
|
|
1039
|
+
return pending;
|
|
1040
|
+
};
|
|
1041
|
+
const defaultRef = await ensure(spec.environment);
|
|
1042
|
+
const byCase = new Map();
|
|
1043
|
+
for (const engineCase of engineCases) {
|
|
1044
|
+
const runtime = resolveEngineCaseExecutionConfig({
|
|
1045
|
+
spec,
|
|
1046
|
+
engineCase: engineCase.case,
|
|
1047
|
+
}).environment;
|
|
1048
|
+
byCase.set(engineCase.id, await ensure(runtime));
|
|
1049
|
+
}
|
|
1050
|
+
return { defaultRef, byCase };
|
|
1051
|
+
}
|
|
1052
|
+
async function ensureLocalDockerfileEnvironment(workspace, spec, runtime) {
|
|
1053
|
+
const dockerfilePath = runtime.dockerfile;
|
|
1054
|
+
const absoluteDockerfile = resolveProjectPath(workspace, dockerfilePath);
|
|
1055
|
+
const rawDockerfile = await fs.readFile(absoluteDockerfile, "utf8").catch((error) => {
|
|
1056
|
+
throw error;
|
|
1057
|
+
});
|
|
1058
|
+
const adapters = await resolveWorkbenchAdaptersForProject(workspace, spec);
|
|
1059
|
+
const dockerfile = await composeRuntimeDockerfileWithAdapters(rawDockerfile, adapters);
|
|
1060
|
+
const digest = createHash("sha256")
|
|
1061
|
+
.update(absoluteDockerfile)
|
|
1062
|
+
.update("\0")
|
|
1063
|
+
.update(dockerfile)
|
|
1064
|
+
.digest("hex")
|
|
1065
|
+
.slice(0, 16);
|
|
1066
|
+
const tag = `workbench-local/${safeDockerTagSegment(spec.name)}:${digest}`;
|
|
1067
|
+
const exists = await spawnOutput("docker", ["image", "inspect", tag]).then(() => true, () => false);
|
|
1068
|
+
if (!exists) {
|
|
1069
|
+
const contextRoot = await fs.mkdtemp(path.join(os.tmpdir(), "workbench-local-runtime-"));
|
|
1070
|
+
const composedDockerfile = path.join(contextRoot, "Dockerfile");
|
|
1071
|
+
await fs.writeFile(composedDockerfile, dockerfile);
|
|
1072
|
+
await spawnOutput("docker", [
|
|
1073
|
+
"build",
|
|
1074
|
+
"-t",
|
|
1075
|
+
tag,
|
|
1076
|
+
"-f",
|
|
1077
|
+
composedDockerfile,
|
|
1078
|
+
contextRoot,
|
|
1079
|
+
]).finally(async () => {
|
|
1080
|
+
await fs.rm(contextRoot, { recursive: true, force: true }).catch(() => undefined);
|
|
1081
|
+
});
|
|
1082
|
+
}
|
|
1083
|
+
return `docker://${tag}`;
|
|
1084
|
+
}
|
|
1085
|
+
async function spawnOutput(command, args) {
|
|
1086
|
+
await new Promise((resolve, reject) => {
|
|
1087
|
+
const child = spawn(command, [...args], {
|
|
1088
|
+
stdio: "ignore",
|
|
1089
|
+
});
|
|
1090
|
+
child.on("error", reject);
|
|
1091
|
+
child.on("exit", (code) => {
|
|
1092
|
+
if (code === 0) {
|
|
1093
|
+
resolve();
|
|
1094
|
+
}
|
|
1095
|
+
else {
|
|
1096
|
+
reject(new Error(`${command} ${args.join(" ")} exited with status ${code ?? "unknown"}.`));
|
|
1097
|
+
}
|
|
1098
|
+
});
|
|
1099
|
+
});
|
|
1100
|
+
}
|
|
1101
|
+
function safeDockerTagSegment(value) {
|
|
1102
|
+
return (value
|
|
1103
|
+
.toLowerCase()
|
|
1104
|
+
.replace(/[^a-z0-9_.-]+/gu, "-")
|
|
1105
|
+
.replace(/^-+|-+$/gu, "") || "workbench");
|
|
1106
|
+
}
|
|
1107
|
+
function requireValidRunEnvelope(args) {
|
|
1108
|
+
const issue = validateWorkbenchRunEnvelope(args);
|
|
1109
|
+
if (issue) {
|
|
1110
|
+
throw new UsageError(issue);
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
async function localCheckpoint(argv, io) {
|
|
1114
|
+
const parsed = parseArgs(argv);
|
|
1115
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
1116
|
+
const workspace = resolveDir(parsed);
|
|
1117
|
+
const projectSource = await readLocalProjectSource(workspace);
|
|
1118
|
+
const spec = projectSource.spec;
|
|
1119
|
+
const subjectRoot = spec.subject.files.path;
|
|
1120
|
+
let snapshot = await loadLocalArchive(workspace);
|
|
1121
|
+
const previous = snapshot.activeId
|
|
1122
|
+
? readLocalSubject(snapshot, snapshot.activeId)
|
|
1123
|
+
: null;
|
|
1124
|
+
const files = normalizeSurfaceFiles(await readSnapshotFiles(resolveProjectPath(workspace, subjectRoot)));
|
|
1125
|
+
const now = new Date().toISOString();
|
|
1126
|
+
const subject = {
|
|
1127
|
+
id: `chk_${Date.now().toString(36)}`,
|
|
1128
|
+
ordinal: snapshot.subjects.length,
|
|
1129
|
+
benchmarkFingerprint: await readLocalBenchmarkFingerprint(workspace),
|
|
1130
|
+
subjectFingerprint: checkpointSubjectFingerprint(files),
|
|
1131
|
+
createdAt: now,
|
|
1132
|
+
...(previous ? { baseId: previous.id } : {}),
|
|
1133
|
+
referenceIds: [],
|
|
1134
|
+
status: "checkpointed",
|
|
1135
|
+
fileChanges: files.map((file) => file.path),
|
|
1136
|
+
};
|
|
1137
|
+
snapshot = upsertLocalSubject(snapshot, subject, files);
|
|
1138
|
+
snapshot = setLocalActive(snapshot, subject.id);
|
|
1139
|
+
await saveLocalArchive(workspace, snapshot);
|
|
1140
|
+
writeOutput({
|
|
1141
|
+
ok: true,
|
|
1142
|
+
activeBefore: previous?.id ?? null,
|
|
1143
|
+
activeAfter: subject.id,
|
|
1144
|
+
changedPaths: subject.fileChanges,
|
|
1145
|
+
}, parsed, io, () => `Checkpointed ${subject.id} with ${subject.fileChanges.length} file(s).`);
|
|
1146
|
+
return 0;
|
|
1147
|
+
}
|
|
1148
|
+
async function localRestore(argv, io) {
|
|
1149
|
+
const parsed = parseArgs(argv);
|
|
1150
|
+
rejectUnknownFlags(parsed, new Set(["dir", "subject", "dry-run", "yes", "json"]));
|
|
1151
|
+
const workspace = resolveDir(parsed);
|
|
1152
|
+
const spec = await readLocalSpecIfValid(workspace);
|
|
1153
|
+
if (!spec) {
|
|
1154
|
+
throw new UsageError("restore requires a valid Workbench project.");
|
|
1155
|
+
}
|
|
1156
|
+
const subjectRoot = spec.subject.files.path;
|
|
1157
|
+
const snapshot = await loadLocalArchive(workspace);
|
|
1158
|
+
const subjectId = readSubjectIdFlag(parsed, snapshot);
|
|
1159
|
+
const files = readLocalSubjectFiles(snapshot, subjectId);
|
|
1160
|
+
if (parsed.flags["dry-run"] === true) {
|
|
1161
|
+
writeOutput({ ok: true, subjectId, fileCount: files.length }, parsed, io, () => `Restore would write ${files.length} file(s) from ${subjectId}.`);
|
|
1162
|
+
return 0;
|
|
1163
|
+
}
|
|
1164
|
+
if (parsed.flags.yes !== true) {
|
|
1165
|
+
throw new UsageError("restore requires --dry-run to preview or --yes to apply source directory changes.");
|
|
1166
|
+
}
|
|
1167
|
+
const changedPaths = await materializeSubjectRoot(workspace, subjectRoot, files);
|
|
1168
|
+
const next = setLocalActive(snapshot, subjectId);
|
|
1169
|
+
await saveLocalArchive(workspace, next);
|
|
1170
|
+
writeOutput({ ok: true, activeAfter: subjectId, changedPaths }, parsed, io, () => `Restored ${subjectId} to ${subjectRoot}.`);
|
|
1171
|
+
return 0;
|
|
1172
|
+
}
|
|
1173
|
+
async function localSubjectList(argv, io) {
|
|
1174
|
+
const parsed = parseArgs(argv);
|
|
1175
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
1176
|
+
const snapshot = await loadLocalArchive(resolveDir(parsed));
|
|
1177
|
+
writeOutput(snapshot.subjects, parsed, io, (subjects) => subjects
|
|
1178
|
+
.map((subject) => `${subject.id}\t${subject.status}\tmetrics ${formatMetricSummary(subject.metrics)}${snapshot.activeId === subject.id ? "\tactive" : ""}`)
|
|
1179
|
+
.join("\n") || "No subjects.");
|
|
1180
|
+
return 0;
|
|
1181
|
+
}
|
|
1182
|
+
async function localSubjectShow(argv, io) {
|
|
1183
|
+
const parsed = parseArgs(argv);
|
|
1184
|
+
rejectUnknownFlags(parsed, new Set(["dir", "subject", "json"]));
|
|
1185
|
+
const snapshot = await loadLocalArchive(resolveDir(parsed));
|
|
1186
|
+
const subjectId = readSubjectIdFlag(parsed, snapshot);
|
|
1187
|
+
const subject = readLocalSubject(snapshot, subjectId);
|
|
1188
|
+
writeOutput(subject, parsed, io, (record) => [
|
|
1189
|
+
`${record.id}\t${record.status}`,
|
|
1190
|
+
`benchmark\t${record.benchmarkFingerprint}`,
|
|
1191
|
+
`subject\t${record.subjectFingerprint}`,
|
|
1192
|
+
`metrics\t${formatMetricSummary(record.metrics)}`,
|
|
1193
|
+
...(record.baseId ? [`base\t${record.baseId}`] : []),
|
|
1194
|
+
].join("\n"));
|
|
1195
|
+
return 0;
|
|
1196
|
+
}
|
|
1197
|
+
async function localSubjectFiles(argv, io) {
|
|
1198
|
+
const parsed = parseArgs(argv);
|
|
1199
|
+
rejectUnknownFlags(parsed, new Set(["dir", "subject", "json"]));
|
|
1200
|
+
const snapshot = await loadLocalArchive(resolveDir(parsed));
|
|
1201
|
+
const subjectId = readSubjectIdFlag(parsed, snapshot);
|
|
1202
|
+
const subject = readLocalSubject(snapshot, subjectId);
|
|
1203
|
+
const files = summarizeSubjectFiles(readLocalSubjectFiles(snapshot, subjectId), subject.fileChanges);
|
|
1204
|
+
writeOutput(files, parsed, io, (records) => records
|
|
1205
|
+
.map((file) => `${file.path}\t${file.status}\t${file.preview_kind}`)
|
|
1206
|
+
.join("\n") || "No files.");
|
|
1207
|
+
return 0;
|
|
1208
|
+
}
|
|
1209
|
+
async function localSubjectPreview(argv, io) {
|
|
1210
|
+
const parsed = parseArgs(argv);
|
|
1211
|
+
rejectUnknownFlags(parsed, new Set(["dir", "subject", "path", "output", "view", "json"]));
|
|
1212
|
+
const snapshot = await loadLocalArchive(resolveDir(parsed));
|
|
1213
|
+
const subjectId = readSubjectIdFlag(parsed, snapshot);
|
|
1214
|
+
const preview = createSubjectFilePreview({
|
|
1215
|
+
files: readLocalSubjectFiles(snapshot, subjectId),
|
|
1216
|
+
path: requireFlag(parsed, "path"),
|
|
1217
|
+
view: readPreviewMode(parsed),
|
|
1218
|
+
});
|
|
1219
|
+
const content = preview.source?.content ?? preview.rendered_html ?? preview.diff ?? "";
|
|
1220
|
+
const outputPath = asOptionalString(parsed.flags.output);
|
|
1221
|
+
if (outputPath && outputPath !== "-") {
|
|
1222
|
+
await fs.writeFile(outputPath, content);
|
|
1223
|
+
io.stdout.write(`Wrote preview to ${outputPath}\n`);
|
|
1224
|
+
}
|
|
1225
|
+
else if (parsed.flags.json === true) {
|
|
1226
|
+
writeJson(preview, io);
|
|
1227
|
+
}
|
|
1228
|
+
else {
|
|
1229
|
+
io.stdout.write(content);
|
|
1230
|
+
}
|
|
1231
|
+
return 0;
|
|
1232
|
+
}
|
|
1233
|
+
async function localRunList(argv, io) {
|
|
1234
|
+
const parsed = parseArgs(argv);
|
|
1235
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
1236
|
+
const snapshot = await loadLocalArchive(resolveDir(parsed));
|
|
1237
|
+
writeOutput(snapshot.runs, parsed, io, (runs) => runs
|
|
1238
|
+
.map((run) => `${run.id}\t${run.workflow}\t${run.status}\t${run.outcome ?? "pending"}\t${run.attemptsExecuted ?? 0}/${run.attemptsRequested ?? 0}`)
|
|
1239
|
+
.join("\n") || "No runs.");
|
|
1240
|
+
return 0;
|
|
1241
|
+
}
|
|
1242
|
+
async function localRunShow(argv, io) {
|
|
1243
|
+
const parsed = parseArgs(argv);
|
|
1244
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
1245
|
+
const runId = parsed.positionals[0];
|
|
1246
|
+
if (!runId) {
|
|
1247
|
+
throw new UsageError("workbench runs show requires RUN_ID.");
|
|
1248
|
+
}
|
|
1249
|
+
const snapshot = await loadLocalArchive(resolveDir(parsed));
|
|
1250
|
+
const run = snapshot.runs.find((entry) => entry.id === runId);
|
|
1251
|
+
if (!run) {
|
|
1252
|
+
throw new UsageError(`Run not found: ${runId}`);
|
|
1253
|
+
}
|
|
1254
|
+
writeOutput(run, parsed, io, (record) => [
|
|
1255
|
+
`${record.id}\t${record.workflow}\t${record.status}`,
|
|
1256
|
+
`outcome\t${record.outcome ?? "pending"}`,
|
|
1257
|
+
`started\t${record.startedAt}`,
|
|
1258
|
+
...(record.finishedAt ? [`finished\t${record.finishedAt}`] : []),
|
|
1259
|
+
`attempts\t${record.attemptsExecuted ?? 0}/${record.attemptsRequested ?? 0}`,
|
|
1260
|
+
`samples\t${record.samples ?? 0}`,
|
|
1261
|
+
].join("\n"));
|
|
1262
|
+
return 0;
|
|
1263
|
+
}
|
|
1264
|
+
async function runAuthCommand(argv, io) {
|
|
1265
|
+
const command = argv[0];
|
|
1266
|
+
const rest = argv.slice(1);
|
|
1267
|
+
switch (command) {
|
|
1268
|
+
case "connect":
|
|
1269
|
+
return await authConnect(rest, io);
|
|
1270
|
+
case "disconnect":
|
|
1271
|
+
return await authDisconnect(rest, io);
|
|
1272
|
+
default:
|
|
1273
|
+
throw new UsageError(`Unknown command: auth ${argv.join(" ")}`);
|
|
1274
|
+
}
|
|
1275
|
+
}
|
|
1276
|
+
async function runAdaptersCommand(argv, io) {
|
|
1277
|
+
const command = argv[0];
|
|
1278
|
+
const rest = argv.slice(1);
|
|
1279
|
+
switch (command) {
|
|
1280
|
+
case "create":
|
|
1281
|
+
return await adaptersCreate(rest, io);
|
|
1282
|
+
case "list":
|
|
1283
|
+
return await adaptersList(rest, io);
|
|
1284
|
+
case "inspect":
|
|
1285
|
+
return await adaptersInspect(rest, io);
|
|
1286
|
+
case "test":
|
|
1287
|
+
return await adaptersTest(rest, io);
|
|
1288
|
+
default:
|
|
1289
|
+
throw new UsageError(`Unknown command: adapters ${argv.join(" ")}`);
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
async function adaptersCreate(argv, io) {
|
|
1293
|
+
const parsed = parseArgs(argv);
|
|
1294
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
1295
|
+
const dir = resolveDir(parsed);
|
|
1296
|
+
const target = parsed.positionals[0];
|
|
1297
|
+
if (!target || parsed.positionals.length > 1) {
|
|
1298
|
+
throw new UsageError("workbench adapters create requires exactly one target directory.");
|
|
1299
|
+
}
|
|
1300
|
+
const absolute = path.resolve(dir, target);
|
|
1301
|
+
const relative = path.relative(dir, absolute);
|
|
1302
|
+
if (relative.startsWith("..") || path.isAbsolute(relative)) {
|
|
1303
|
+
throw new UsageError("Adapter create target must be inside the benchmark source root.");
|
|
1304
|
+
}
|
|
1305
|
+
const id = path.basename(absolute).trim().toLowerCase().replace(/[^a-z0-9]+/gu, "-").replace(/^-+|-+$/gu, "");
|
|
1306
|
+
if (!/^[a-z][a-z0-9-]*$/u.test(id)) {
|
|
1307
|
+
throw new UsageError("Adapter directory name must produce a lowercase adapter id.");
|
|
1308
|
+
}
|
|
1309
|
+
const files = createAdapterScaffoldFiles(id);
|
|
1310
|
+
for (const file of files) {
|
|
1311
|
+
const destination = path.join(absolute, file.path);
|
|
1312
|
+
await fs.mkdir(path.dirname(destination), { recursive: true });
|
|
1313
|
+
await fs.writeFile(destination, file.content, { mode: file.executable ? 0o755 : 0o644 });
|
|
1314
|
+
}
|
|
1315
|
+
writeOutput({
|
|
1316
|
+
ok: true,
|
|
1317
|
+
id,
|
|
1318
|
+
path: absolute,
|
|
1319
|
+
files: files.map((file) => file.path),
|
|
1320
|
+
}, parsed, io, (record) => {
|
|
1321
|
+
const value = record;
|
|
1322
|
+
return `Created adapter ${value.id} at ${value.path} (${value.files.length} file(s)).`;
|
|
1323
|
+
});
|
|
1324
|
+
return 0;
|
|
1325
|
+
}
|
|
1326
|
+
async function adaptersList(argv, io) {
|
|
1327
|
+
const parsed = parseArgs(argv);
|
|
1328
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
1329
|
+
const dir = resolveDir(parsed);
|
|
1330
|
+
const projectSource = await readLocalProjectSourceIfPresent(dir);
|
|
1331
|
+
const projectAdapters = projectSource
|
|
1332
|
+
? await resolveWorkbenchAdaptersForProject(dir, projectSource.spec)
|
|
1333
|
+
: [];
|
|
1334
|
+
const projectAdaptersById = new Map(projectAdapters.map((adapter) => [adapter.manifest.id, adapter]));
|
|
1335
|
+
const defaults = defaultAdapterManifests()
|
|
1336
|
+
.filter((manifest) => !projectAdaptersById.has(manifest.id))
|
|
1337
|
+
.map((manifest) => ({
|
|
1338
|
+
id: manifest.id,
|
|
1339
|
+
declaredSource: `default:${manifest.id}`,
|
|
1340
|
+
resolvedSource: `default:${manifest.id}`,
|
|
1341
|
+
kind: "default",
|
|
1342
|
+
stability: "default",
|
|
1343
|
+
installed: false,
|
|
1344
|
+
operations: adapterOperationCommands(manifest.operations),
|
|
1345
|
+
}));
|
|
1346
|
+
const project = projectAdapters
|
|
1347
|
+
.map((adapter) => ({
|
|
1348
|
+
id: adapter.manifest.id,
|
|
1349
|
+
kind: adapter.kind,
|
|
1350
|
+
declaredSource: adapter.declaredSource,
|
|
1351
|
+
resolvedSource: adapter.source,
|
|
1352
|
+
stability: adapter.stability,
|
|
1353
|
+
installed: true,
|
|
1354
|
+
operations: adapterOperationCommands(adapter.manifest.operations),
|
|
1355
|
+
...(adapter.overridesDefault ? { overridesDefault: true } : {}),
|
|
1356
|
+
}));
|
|
1357
|
+
const adapters = [...defaults, ...project].sort((left, right) => left.id.localeCompare(right.id));
|
|
1358
|
+
writeOutput({ ok: true, adapters }, parsed, io, (record) => {
|
|
1359
|
+
const value = record;
|
|
1360
|
+
return value.adapters.map((adapter) => `${adapter.id}\t${adapter.installed ? "installed" : "available"}\t${adapter.stability}${adapter.overridesDefault ? " override" : ""}\t${formatAdapterResolution(adapter)}`).join("\n");
|
|
1361
|
+
});
|
|
1362
|
+
return 0;
|
|
1363
|
+
}
|
|
1364
|
+
async function adaptersInspect(argv, io) {
|
|
1365
|
+
const parsed = parseArgs(argv);
|
|
1366
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
1367
|
+
const dir = resolveDir(parsed);
|
|
1368
|
+
const id = parsed.positionals[0];
|
|
1369
|
+
if (!id || parsed.positionals.length > 1) {
|
|
1370
|
+
throw new UsageError("workbench adapters inspect requires exactly one adapter id.");
|
|
1371
|
+
}
|
|
1372
|
+
const projectSource = await readLocalProjectSourceIfPresent(dir);
|
|
1373
|
+
const projectAdapters = projectSource
|
|
1374
|
+
? await resolveWorkbenchAdaptersForProject(dir, projectSource.spec)
|
|
1375
|
+
: [];
|
|
1376
|
+
const adapter = projectAdapters.find((entry) => entry.manifest.id === id || entry.declaredSource === id || entry.source === id) ??
|
|
1377
|
+
resolveDefaultWorkbenchAdapter(id);
|
|
1378
|
+
if (!adapter) {
|
|
1379
|
+
throw new UsageError(`Adapter ${id} is not installed or available in the default catalog.`);
|
|
1380
|
+
}
|
|
1381
|
+
writeOutput({
|
|
1382
|
+
ok: true,
|
|
1383
|
+
adapter: adapterRecordForOutput(adapter),
|
|
1384
|
+
}, parsed, io, (record) => {
|
|
1385
|
+
const value = record;
|
|
1386
|
+
const setup = value.adapter.setup.length > 0 ? value.adapter.setup.join("; ") : "none";
|
|
1387
|
+
const override = value.adapter.overridesDefault ? "overrides default" : value.adapter.kind;
|
|
1388
|
+
const operations = Object.entries(value.adapter.operations)
|
|
1389
|
+
.map(([operation, command]) => `${operation}: ${command}`)
|
|
1390
|
+
.join("; ");
|
|
1391
|
+
return [
|
|
1392
|
+
`${value.adapter.id} (${formatAdapterResolution(value.adapter)}, ${value.adapter.stability}, ${override})`,
|
|
1393
|
+
`operations: ${operations || "none"}`,
|
|
1394
|
+
`setup: ${setup}`,
|
|
1395
|
+
`auth: ${value.adapter.auth ? "declared" : "none"}`,
|
|
1396
|
+
].join("\n");
|
|
1397
|
+
});
|
|
1398
|
+
return 0;
|
|
1399
|
+
}
|
|
1400
|
+
async function adaptersTest(argv, io) {
|
|
1401
|
+
const parsed = parseArgs(argv);
|
|
1402
|
+
rejectUnknownFlags(parsed, new Set(["dir", "request", "output", "json"]));
|
|
1403
|
+
const dir = resolveDir(parsed);
|
|
1404
|
+
const target = parsed.positionals[0];
|
|
1405
|
+
if (!target || parsed.positionals.length > 1) {
|
|
1406
|
+
throw new UsageError("workbench adapters test requires exactly one adapter id or source.");
|
|
1407
|
+
}
|
|
1408
|
+
const adapter = await resolveAdapterForAdaptersTest(dir, target);
|
|
1409
|
+
const requestArg = asOptionalString(parsed.flags.request);
|
|
1410
|
+
const replay = requestArg
|
|
1411
|
+
? await runAdapterTestReplay({
|
|
1412
|
+
adapter,
|
|
1413
|
+
dir,
|
|
1414
|
+
requestPath: path.resolve(dir, requestArg),
|
|
1415
|
+
outputRoot: asOptionalString(parsed.flags.output),
|
|
1416
|
+
})
|
|
1417
|
+
: null;
|
|
1418
|
+
writeOutput({
|
|
1419
|
+
ok: true,
|
|
1420
|
+
mode: replay ? "replay" : "manifest",
|
|
1421
|
+
adapter: adapterRecordForOutput(adapter),
|
|
1422
|
+
...(replay ? { replay } : {}),
|
|
1423
|
+
}, parsed, io, (record) => {
|
|
1424
|
+
const value = record;
|
|
1425
|
+
if (value.mode === "manifest") {
|
|
1426
|
+
return `Adapter ${value.adapter.id} manifest is valid (${formatAdapterResolution(value.adapter)}).`;
|
|
1427
|
+
}
|
|
1428
|
+
return [
|
|
1429
|
+
`Adapter ${value.adapter.id} replay passed (${value.replay?.operation ?? "unknown"}, ${value.replay?.outputs.length ?? 0} output(s)).`,
|
|
1430
|
+
`output: ${value.replay?.outputRoot ?? ""}`,
|
|
1431
|
+
].join("\n");
|
|
1432
|
+
});
|
|
1433
|
+
return 0;
|
|
1434
|
+
}
|
|
1435
|
+
async function resolveAdapterForAdaptersTest(dir, target) {
|
|
1436
|
+
const projectSource = await readLocalProjectSourceIfPresent(dir);
|
|
1437
|
+
if (projectSource) {
|
|
1438
|
+
const adapters = await resolveWorkbenchAdaptersForProject(dir, projectSource.spec);
|
|
1439
|
+
const adapter = adapters.find((entry) => entry.manifest.id === target || entry.declaredSource === target || entry.source === target);
|
|
1440
|
+
if (adapter) {
|
|
1441
|
+
return adapter;
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
if (isAdapterSourceTarget(target)) {
|
|
1445
|
+
return await resolveProjectAdapterSource(dir, target);
|
|
1446
|
+
}
|
|
1447
|
+
const defaultAdapter = resolveDefaultWorkbenchAdapter(target);
|
|
1448
|
+
if (defaultAdapter) {
|
|
1449
|
+
return defaultAdapter;
|
|
1450
|
+
}
|
|
1451
|
+
throw new UsageError(`Adapter ${target} is not installed, available in the default catalog, or resolvable as a source.`);
|
|
1452
|
+
}
|
|
1453
|
+
function isAdapterSourceTarget(target) {
|
|
1454
|
+
return target.startsWith("npm:") ||
|
|
1455
|
+
target.startsWith("git:") ||
|
|
1456
|
+
target.startsWith(".") ||
|
|
1457
|
+
target.startsWith("/") ||
|
|
1458
|
+
target.includes("/");
|
|
1459
|
+
}
|
|
1460
|
+
async function runAdapterTestReplay(args) {
|
|
1461
|
+
const request = normalizeWorkbenchAdapterOperationRequest(JSON.parse(await fs.readFile(args.requestPath, "utf8")));
|
|
1462
|
+
if (request.invocation.use !== args.adapter.manifest.id) {
|
|
1463
|
+
throw new Error(`Request invocation.use ${request.invocation.use} does not match adapter id ${args.adapter.manifest.id}.`);
|
|
1464
|
+
}
|
|
1465
|
+
const outputRoot = args.outputRoot
|
|
1466
|
+
? path.resolve(args.dir, args.outputRoot)
|
|
1467
|
+
: await fs.mkdtemp(path.join(os.tmpdir(), "workbench-adapter-output-"));
|
|
1468
|
+
await fs.mkdir(outputRoot, { recursive: true });
|
|
1469
|
+
const replayRequest = adapterTestRequestForOutput(request, outputRoot);
|
|
1470
|
+
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "workbench-adapter-test-"));
|
|
1471
|
+
const runtimeRequestPath = path.join(tempRoot, "request.json");
|
|
1472
|
+
try {
|
|
1473
|
+
await fs.writeFile(runtimeRequestPath, `${JSON.stringify(replayRequest, null, 2)}\n`);
|
|
1474
|
+
const commandOutput = await runAdapterCommandForTest({
|
|
1475
|
+
adapter: args.adapter,
|
|
1476
|
+
cwd: adapterCommandCwd(args.adapter, args.dir),
|
|
1477
|
+
requestPath: runtimeRequestPath,
|
|
1478
|
+
outputRoot,
|
|
1479
|
+
});
|
|
1480
|
+
const outputs = await validateAdapterTestOutputs(replayRequest, outputRoot);
|
|
1481
|
+
const command = workbenchAdapterOperationCommand(args.adapter.manifest, replayRequest.operation);
|
|
1482
|
+
return {
|
|
1483
|
+
requestPath: args.requestPath,
|
|
1484
|
+
outputRoot,
|
|
1485
|
+
operation: replayRequest.operation,
|
|
1486
|
+
command,
|
|
1487
|
+
stdout: commandOutput.stdout,
|
|
1488
|
+
stderr: commandOutput.stderr,
|
|
1489
|
+
outputs,
|
|
1490
|
+
};
|
|
1491
|
+
}
|
|
1492
|
+
finally {
|
|
1493
|
+
await fs.rm(tempRoot, { recursive: true, force: true }).catch(() => undefined);
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
function adapterTestRequestForOutput(request, outputRoot) {
|
|
1497
|
+
return {
|
|
1498
|
+
...request,
|
|
1499
|
+
paths: {
|
|
1500
|
+
...request.paths,
|
|
1501
|
+
output: outputRoot,
|
|
1502
|
+
result: workbenchAdapterOperationResultPath(outputRoot),
|
|
1503
|
+
},
|
|
1504
|
+
};
|
|
1505
|
+
}
|
|
1506
|
+
function adapterCommandCwd(adapter, fallback) {
|
|
1507
|
+
return adapter.root ?? fallback;
|
|
1508
|
+
}
|
|
1509
|
+
async function runAdapterCommandForTest(args) {
|
|
1510
|
+
const env = adapterTestEnv(args.requestPath, args.outputRoot);
|
|
1511
|
+
const command = workbenchAdapterOperationCommand(args.adapter.manifest, normalizeWorkbenchAdapterOperationRequest(JSON.parse(await fs.readFile(args.requestPath, "utf8"))).operation);
|
|
1512
|
+
return await runShellCommand({
|
|
1513
|
+
command,
|
|
1514
|
+
cwd: args.cwd,
|
|
1515
|
+
env,
|
|
1516
|
+
errorLabel: `Adapter command ${command}`,
|
|
1517
|
+
});
|
|
1518
|
+
}
|
|
1519
|
+
function adapterTestEnv(requestPath, outputRoot) {
|
|
1520
|
+
const env = {};
|
|
1521
|
+
for (const [key, value] of Object.entries(process.env)) {
|
|
1522
|
+
if (typeof value === "string" && !key.startsWith("WORKBENCH_")) {
|
|
1523
|
+
env[key] = value;
|
|
1524
|
+
}
|
|
1525
|
+
}
|
|
1526
|
+
env.PATH = process.env.PATH
|
|
1527
|
+
? `${process.env.PATH}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin`
|
|
1528
|
+
: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
1529
|
+
env.WORKBENCH_ADAPTER_REQUEST = requestPath;
|
|
1530
|
+
env.WORKBENCH_OUTPUT = outputRoot;
|
|
1531
|
+
env.WORKBENCH_RESULT = workbenchAdapterOperationResultPath(outputRoot);
|
|
1532
|
+
return env;
|
|
1533
|
+
}
|
|
1534
|
+
async function validateAdapterTestOutputs(request, outputRoot) {
|
|
1535
|
+
await readWorkbenchAdapterOperationResult(outputRoot, request.operation).catch((error) => {
|
|
1536
|
+
if (error.code === "ENOENT") {
|
|
1537
|
+
throw new Error(`Adapter did not write workbench-result.json.`);
|
|
1538
|
+
}
|
|
1539
|
+
throw error;
|
|
1540
|
+
}).then((result) => assertWorkbenchAdapterOperationResultOk(result, `Adapter ${request.invocation.use} ${request.operation}`));
|
|
1541
|
+
return ["workbench-result.json"];
|
|
1542
|
+
}
|
|
1543
|
+
function adapterRecordForOutput(adapter) {
|
|
1544
|
+
return {
|
|
1545
|
+
id: adapter.manifest.id,
|
|
1546
|
+
declaredSource: adapter.declaredSource,
|
|
1547
|
+
resolvedSource: adapter.source,
|
|
1548
|
+
kind: adapter.kind,
|
|
1549
|
+
stability: adapter.stability,
|
|
1550
|
+
operations: adapterOperationCommands(adapter.manifest.operations),
|
|
1551
|
+
setup: [...adapter.manifest.setup],
|
|
1552
|
+
slots: adapter.manifest.slots ?? {},
|
|
1553
|
+
...(adapter.overridesDefault ? { overridesDefault: true } : {}),
|
|
1554
|
+
...(adapter.manifest.auth !== undefined ? { auth: adapter.manifest.auth } : {}),
|
|
1555
|
+
...(adapter.integrity ? { integrity: adapter.integrity } : {}),
|
|
1556
|
+
manifestHash: adapter.manifestHash,
|
|
1557
|
+
contentHash: adapter.contentHash,
|
|
1558
|
+
};
|
|
1559
|
+
}
|
|
1560
|
+
function adapterOperationCommands(operations) {
|
|
1561
|
+
return Object.fromEntries(Object.entries(operations).map(([operation, config]) => [operation, config.command]));
|
|
1562
|
+
}
|
|
1563
|
+
function createAdapterScaffoldFiles(id) {
|
|
1564
|
+
const command = `workbench-adapter-${id}`;
|
|
1565
|
+
const manifest = [
|
|
1566
|
+
`id: ${id}`,
|
|
1567
|
+
"protocol: workbench.adapter.v3",
|
|
1568
|
+
"setup:",
|
|
1569
|
+
" - npm install --global .",
|
|
1570
|
+
"operations:",
|
|
1571
|
+
" subject.run: {}",
|
|
1572
|
+
"",
|
|
1573
|
+
].join("\n");
|
|
1574
|
+
const packageJson = `${JSON.stringify({
|
|
1575
|
+
name: `@local/${id}-workbench-adapter`,
|
|
1576
|
+
version: "0.0.0",
|
|
1577
|
+
type: "module",
|
|
1578
|
+
private: true,
|
|
1579
|
+
bin: {
|
|
1580
|
+
[command]: "./adapter.mjs",
|
|
1581
|
+
},
|
|
1582
|
+
}, null, 2)}\n`;
|
|
1583
|
+
const adapter = `#!/usr/bin/env node
|
|
1584
|
+
import fs from "node:fs";
|
|
1585
|
+
import path from "node:path";
|
|
1586
|
+
|
|
1587
|
+
const requestPath = process.env.WORKBENCH_ADAPTER_REQUEST;
|
|
1588
|
+
const outputRoot = process.env.WORKBENCH_OUTPUT || "/workspace/output";
|
|
1589
|
+
const request = requestPath && fs.existsSync(requestPath)
|
|
1590
|
+
? JSON.parse(fs.readFileSync(requestPath, "utf8"))
|
|
1591
|
+
: {};
|
|
1592
|
+
fs.mkdirSync(outputRoot, { recursive: true });
|
|
1593
|
+
const operation = request.operation || "subject.run";
|
|
1594
|
+
const resultPath = process.env.WORKBENCH_RESULT || request.paths?.result || path.join(outputRoot, "workbench-result.json");
|
|
1595
|
+
|
|
1596
|
+
let value;
|
|
1597
|
+
if (operation === "subject.run") {
|
|
1598
|
+
const task = request.context?.case?.prompt || "No case prompt was provided.";
|
|
1599
|
+
fs.writeFileSync(path.join(outputRoot, "adapter-output.txt"), [
|
|
1600
|
+
"adapter: ${id}",
|
|
1601
|
+
"task:",
|
|
1602
|
+
task,
|
|
1603
|
+
"",
|
|
1604
|
+
].join("\\n"));
|
|
1605
|
+
} else {
|
|
1606
|
+
console.error("${id} only implements subject.run.");
|
|
1607
|
+
process.exit(2);
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
fs.writeFileSync(resultPath, JSON.stringify({
|
|
1611
|
+
protocol: "workbench.adapter-result.v1",
|
|
1612
|
+
operation,
|
|
1613
|
+
ok: true,
|
|
1614
|
+
...(value === undefined ? {} : { value }),
|
|
1615
|
+
summary: "${id} completed " + operation + ".",
|
|
1616
|
+
}, null, 2) + "\\n");
|
|
1617
|
+
`;
|
|
1618
|
+
const readme = [
|
|
1619
|
+
`# ${id}`,
|
|
1620
|
+
"",
|
|
1621
|
+
"This is a Workbench adapter. It receives a JSON request at `WORKBENCH_ADAPTER_REQUEST` and writes phase outputs under `WORKBENCH_OUTPUT`.",
|
|
1622
|
+
"",
|
|
1623
|
+
"Validate the manifest with `workbench adapters test PATH`. Replay a request fixture with `workbench adapters test PATH --request adapter-request.json --output out/adapter-test`.",
|
|
1624
|
+
"",
|
|
1625
|
+
"See `docs/evals/adapters.md` in the Workbench source for the full adapter contract.",
|
|
1626
|
+
"",
|
|
1627
|
+
].join("\n");
|
|
1628
|
+
return [
|
|
1629
|
+
{ path: WORKBENCH_ADAPTER_MANIFEST_FILE, content: manifest },
|
|
1630
|
+
{ path: "package.json", content: packageJson },
|
|
1631
|
+
{ path: "adapter.mjs", content: adapter, executable: true },
|
|
1632
|
+
{ path: "README.md", content: readme },
|
|
1633
|
+
];
|
|
1634
|
+
}
|
|
1635
|
+
async function login(argv, io) {
|
|
1636
|
+
const parsed = parseArgs(argv);
|
|
1637
|
+
rejectUnknownFlags(parsed, new Set(["base-url", "no-open", "json"]));
|
|
1638
|
+
const baseUrl = asOptionalString(parsed.flags["base-url"]) ?? DEFAULT_BASE_URL;
|
|
1639
|
+
const authorization = await requestDeviceAuthorization(baseUrl);
|
|
1640
|
+
if (parsed.flags.json === true) {
|
|
1641
|
+
writeJson({ ok: true, status: "authorization_pending", ...authorization }, io);
|
|
1642
|
+
}
|
|
1643
|
+
else {
|
|
1644
|
+
io.stdout.write(`Open ${authorization.verification_uri_complete}\n`);
|
|
1645
|
+
io.stdout.write(`Code: ${authorization.user_code}\n`);
|
|
1646
|
+
}
|
|
1647
|
+
if (parsed.flags["no-open"] !== true) {
|
|
1648
|
+
await openBrowser(authorization.verification_uri_complete).catch(() => undefined);
|
|
1649
|
+
}
|
|
1650
|
+
const token = await pollDeviceToken(baseUrl, authorization);
|
|
1651
|
+
await writeConfig({ baseUrl, accessToken: token.access_token });
|
|
1652
|
+
if (parsed.flags.json === true) {
|
|
1653
|
+
writeJson({ ok: true, baseUrl, expiresIn: token.expires_in }, io);
|
|
1654
|
+
}
|
|
1655
|
+
else {
|
|
1656
|
+
io.stdout.write(`Workbench API: ${baseUrl}\n`);
|
|
1657
|
+
}
|
|
1658
|
+
return 0;
|
|
1659
|
+
}
|
|
1660
|
+
async function logout(argv, io) {
|
|
1661
|
+
const parsed = parseArgs(argv);
|
|
1662
|
+
rejectUnknownFlags(parsed, new Set(["json"]));
|
|
1663
|
+
const config = await loadConfig();
|
|
1664
|
+
const baseUrl = normalizeBaseUrl(process.env.WORKBENCH_API_URL ?? config.baseUrl ?? DEFAULT_BASE_URL);
|
|
1665
|
+
if (config.accessToken) {
|
|
1666
|
+
await fetch(`${baseUrl}/api/oauth/revoke`, {
|
|
1667
|
+
method: "POST",
|
|
1668
|
+
headers: {
|
|
1669
|
+
"content-type": "application/json",
|
|
1670
|
+
},
|
|
1671
|
+
body: JSON.stringify({ token: config.accessToken }),
|
|
1672
|
+
}).catch(() => undefined);
|
|
1673
|
+
}
|
|
1674
|
+
await writeConfig({ baseUrl });
|
|
1675
|
+
writeOutput({ ok: true, baseUrl }, parsed, io, () => "Logged out of Workbench.");
|
|
1676
|
+
return 0;
|
|
1677
|
+
}
|
|
1678
|
+
async function authStatus(argv, io) {
|
|
1679
|
+
const parsed = parseArgs(argv);
|
|
1680
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
1681
|
+
const config = await loadConfig();
|
|
1682
|
+
const baseUrl = await effectiveBaseUrl();
|
|
1683
|
+
const profile = config.accessToken
|
|
1684
|
+
? await apiRequest("/api/workbench/profile").catch(() => ({ profile: null }))
|
|
1685
|
+
: { profile: null };
|
|
1686
|
+
const adapterStatuses = await localWorkbenchAdapterAuthStore().listStatus();
|
|
1687
|
+
const hostedAuth = config.accessToken
|
|
1688
|
+
? await readHostedAdapterAuthStatuses().catch((error) => ({
|
|
1689
|
+
adapters: [],
|
|
1690
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1691
|
+
}))
|
|
1692
|
+
: {
|
|
1693
|
+
adapters: [],
|
|
1694
|
+
error: "not_authenticated",
|
|
1695
|
+
};
|
|
1696
|
+
const dir = resolveDir(parsed);
|
|
1697
|
+
const adapterAuth = await projectAdapterAuthStatus(dir, adapterStatuses, hostedAuth.adapters).catch(() => []);
|
|
1698
|
+
const result = {
|
|
1699
|
+
ok: true,
|
|
1700
|
+
workbench: {
|
|
1701
|
+
baseUrl,
|
|
1702
|
+
authenticated: Boolean(config.accessToken),
|
|
1703
|
+
username: profile.profile?.username ?? null,
|
|
1704
|
+
},
|
|
1705
|
+
adapterStatuses,
|
|
1706
|
+
hostedAuth,
|
|
1707
|
+
adapterAuth,
|
|
1708
|
+
};
|
|
1709
|
+
writeOutput(result, parsed, io, (record) => {
|
|
1710
|
+
const value = record;
|
|
1711
|
+
return [
|
|
1712
|
+
`Workbench API: ${value.workbench.baseUrl}`,
|
|
1713
|
+
`Workbench authenticated: ${value.workbench.authenticated ? "yes" : "no"}`,
|
|
1714
|
+
...(value.workbench.username ? [`Username: ${value.workbench.username}`] : []),
|
|
1715
|
+
...(value.adapterAuth.length > 0
|
|
1716
|
+
? [
|
|
1717
|
+
"",
|
|
1718
|
+
"Required adapter auth:",
|
|
1719
|
+
...value.adapterAuth.map((adapter) => `${adapter.adapter}${adapter.profile !== "default" ? ` profile ${adapter.profile}` : ""}: local ${adapter.local.status}${adapter.local.method ? ` (${adapter.local.method})` : ""}${adapter.local.reason ? ` (${adapter.local.reason})` : ""}, hosted ${adapter.hosted.status}${adapter.hosted.method ? ` (${adapter.hosted.method})` : ""}${adapter.hosted.reason ? ` (${adapter.hosted.reason})` : ""}`),
|
|
1720
|
+
]
|
|
1721
|
+
: []),
|
|
1722
|
+
].join("\n");
|
|
1723
|
+
});
|
|
1724
|
+
return 0;
|
|
1725
|
+
}
|
|
1726
|
+
async function projectAdapterAuthStatus(dir, adapterStatuses, hostedAdapters) {
|
|
1727
|
+
const spec = (await readLocalProjectSource(dir)).spec;
|
|
1728
|
+
const adapters = await resolveWorkbenchAdaptersForProject(dir, spec);
|
|
1729
|
+
const adapterStatusMap = new Map(adapterStatuses.map((status) => [
|
|
1730
|
+
adapterAuthStatusKey(status.adapterId, status.slot, status.profile),
|
|
1731
|
+
status,
|
|
1732
|
+
]));
|
|
1733
|
+
const hostedAdapterStatusMap = new Map(hostedAdapters.map((status) => [
|
|
1734
|
+
adapterAuthStatusKey(status.adapterId, status.slot, status.profile),
|
|
1735
|
+
status,
|
|
1736
|
+
]));
|
|
1737
|
+
const adapterById = new Map(adapters.map((adapter) => [adapter.manifest.id, adapter]));
|
|
1738
|
+
return requiredAuthTargetsForSpec(spec, adapterById).map((target) => {
|
|
1739
|
+
const adapterStatus = adapterStatusMap.get(adapterAuthStatusKey(target.adapter, target.slot, target.profile));
|
|
1740
|
+
const hostedAdapterStatus = hostedAdapterStatusMap.get(adapterAuthStatusKey(target.adapter, target.slot, target.profile));
|
|
1741
|
+
return {
|
|
1742
|
+
...target,
|
|
1743
|
+
local: adapterStatus
|
|
1744
|
+
? {
|
|
1745
|
+
status: adapterStatus.status,
|
|
1746
|
+
...(adapterStatus.method ? { method: adapterStatus.method } : {}),
|
|
1747
|
+
...(adapterStatus.reason ? { reason: adapterStatus.reason } : {}),
|
|
1748
|
+
}
|
|
1749
|
+
: { status: "disconnected" },
|
|
1750
|
+
hosted: hostedAdapterStatus
|
|
1751
|
+
? {
|
|
1752
|
+
status: hostedAdapterStatus.status,
|
|
1753
|
+
...(hostedAdapterStatus.method ? { method: hostedAdapterStatus.method } : {}),
|
|
1754
|
+
...(hostedAdapterStatus.reason ? { reason: hostedAdapterStatus.reason } : {}),
|
|
1755
|
+
}
|
|
1756
|
+
: { status: "disconnected" },
|
|
1757
|
+
};
|
|
1758
|
+
});
|
|
1759
|
+
}
|
|
1760
|
+
async function readHostedAdapterAuthStatuses() {
|
|
1761
|
+
const adapterResponse = await apiRequest("/api/workbench/auth/adapters");
|
|
1762
|
+
return {
|
|
1763
|
+
adapters: adapterResponse.adapters ?? [],
|
|
1764
|
+
};
|
|
1765
|
+
}
|
|
1766
|
+
function requiredAuthTargetsForSpec(spec, adapterById) {
|
|
1767
|
+
const manifests = [...adapterById.values()].map((adapter) => adapter.manifest);
|
|
1768
|
+
return collectWorkbenchAdapterAuthRequirements([
|
|
1769
|
+
...(spec.improve ? [spec.improve] : []),
|
|
1770
|
+
spec.run,
|
|
1771
|
+
spec.engineRun,
|
|
1772
|
+
], manifests).map((target) => ({
|
|
1773
|
+
adapter: target.adapterId,
|
|
1774
|
+
...(target.slot ? { slot: target.slot } : {}),
|
|
1775
|
+
profile: target.profile,
|
|
1776
|
+
})).sort((left, right) => adapterAuthStatusKey(left.adapter, left.slot, left.profile)
|
|
1777
|
+
.localeCompare(adapterAuthStatusKey(right.adapter, right.slot, right.profile)));
|
|
1778
|
+
}
|
|
1779
|
+
function adapterAuthStatusKey(adapterId, slot, profile) {
|
|
1780
|
+
return `${adapterId}/${slot ?? "_"}/${profile}`;
|
|
1781
|
+
}
|
|
1782
|
+
async function authConnect(argv, io) {
|
|
1783
|
+
const parsed = parseArgs(argv);
|
|
1784
|
+
rejectUnknownFlags(parsed, new Set([
|
|
1785
|
+
"profile-root",
|
|
1786
|
+
"method",
|
|
1787
|
+
"dir",
|
|
1788
|
+
"profile",
|
|
1789
|
+
"local-only",
|
|
1790
|
+
"json",
|
|
1791
|
+
]));
|
|
1792
|
+
const targetRaw = readAuthTargetPositional(parsed, "connect");
|
|
1793
|
+
const profile = readAuthProfileFlag(parsed);
|
|
1794
|
+
const dir = resolveDir(parsed);
|
|
1795
|
+
const adapter = await resolveAdapterForAuthTarget(dir, targetRaw);
|
|
1796
|
+
const target = parseWorkbenchAdapterAuthTarget(targetRaw, profile);
|
|
1797
|
+
const method = readAdapterConnectMethod(adapter.manifest, target.slot, parsed);
|
|
1798
|
+
const saved = await localWorkbenchAdapterAuthStore().put(await collectAdapterAuthForConnect({
|
|
1799
|
+
adapter,
|
|
1800
|
+
target,
|
|
1801
|
+
targetRaw,
|
|
1802
|
+
method,
|
|
1803
|
+
profileRoot: path.resolve(asOptionalString(parsed.flags["profile-root"]) ?? os.homedir()),
|
|
1804
|
+
cwd: dir,
|
|
1805
|
+
}));
|
|
1806
|
+
const remote = parsed.flags["local-only"] === true
|
|
1807
|
+
? { status: "skipped", reason: "local_only" }
|
|
1808
|
+
: await uploadAdapterConnection(saved);
|
|
1809
|
+
writeOutput({
|
|
1810
|
+
ok: true,
|
|
1811
|
+
adapter: target.adapterId,
|
|
1812
|
+
...(target.slot ? { slot: target.slot } : {}),
|
|
1813
|
+
profile: target.profile,
|
|
1814
|
+
method: saved.method,
|
|
1815
|
+
status: "connected",
|
|
1816
|
+
version: saved.version,
|
|
1817
|
+
updatedAt: saved.updatedAt,
|
|
1818
|
+
remote,
|
|
1819
|
+
}, parsed, io, (record) => {
|
|
1820
|
+
const value = record;
|
|
1821
|
+
const label = value.slot ? `${value.adapter}/${value.slot}` : value.adapter;
|
|
1822
|
+
const profileText = value.profile === "default" ? "" : ` profile ${value.profile}`;
|
|
1823
|
+
const remoteLine = value.remote.status === "connected"
|
|
1824
|
+
? "remote: connected"
|
|
1825
|
+
: `remote: ${value.remote.status}${value.remote.reason ? ` (${value.remote.reason})` : ""}`;
|
|
1826
|
+
return `Connected ${label}${profileText} adapter ${value.method} auth v${value.version}; ${remoteLine}.`;
|
|
1827
|
+
});
|
|
1828
|
+
return 0;
|
|
1829
|
+
}
|
|
1830
|
+
async function authDisconnect(argv, io) {
|
|
1831
|
+
const parsed = parseArgs(argv);
|
|
1832
|
+
rejectUnknownFlags(parsed, new Set(["local-only", "profile", "json"]));
|
|
1833
|
+
const targetRaw = readAuthTargetPositional(parsed, "disconnect");
|
|
1834
|
+
const profile = readAuthProfileFlag(parsed);
|
|
1835
|
+
const target = parseWorkbenchAdapterAuthTarget(targetRaw, profile);
|
|
1836
|
+
await localWorkbenchAdapterAuthStore().disconnect(target);
|
|
1837
|
+
const remote = parsed.flags["local-only"] === true
|
|
1838
|
+
? { status: "skipped", reason: "local_only" }
|
|
1839
|
+
: await deleteAdapterConnection(target);
|
|
1840
|
+
writeOutput({
|
|
1841
|
+
ok: true,
|
|
1842
|
+
adapter: target.adapterId,
|
|
1843
|
+
...(target.slot ? { slot: target.slot } : {}),
|
|
1844
|
+
profile: target.profile,
|
|
1845
|
+
status: "disconnected",
|
|
1846
|
+
remote,
|
|
1847
|
+
}, parsed, io, (record) => {
|
|
1848
|
+
const value = record;
|
|
1849
|
+
const label = value.slot ? `${value.adapter}/${value.slot}` : value.adapter;
|
|
1850
|
+
const remoteLine = value.remote.status === "disconnected"
|
|
1851
|
+
? "remote: disconnected"
|
|
1852
|
+
: `remote: ${value.remote.status}${value.remote.reason ? ` (${value.remote.reason})` : ""}`;
|
|
1853
|
+
return `Disconnected ${label} adapter auth; ${remoteLine}.`;
|
|
1854
|
+
});
|
|
1855
|
+
return 0;
|
|
1856
|
+
}
|
|
1857
|
+
function readAuthTargetPositional(parsed, command) {
|
|
1858
|
+
const target = parsed.positionals[0];
|
|
1859
|
+
if (!target) {
|
|
1860
|
+
throw new UsageError(`workbench auth ${command} requires adapter or adapter/slot.`);
|
|
1861
|
+
}
|
|
1862
|
+
if (parsed.positionals.length > 1) {
|
|
1863
|
+
throw new UsageError(`Unexpected argument for workbench auth ${command}: ${parsed.positionals.slice(1).join(" ")}`);
|
|
1864
|
+
}
|
|
1865
|
+
if (!/^[a-z][a-z0-9-]*(?:\/[a-z][a-z0-9-]*)?$/u.test(target)) {
|
|
1866
|
+
throw new UsageError("Adapter auth target must be adapter or adapter/slot.");
|
|
1867
|
+
}
|
|
1868
|
+
return target;
|
|
1869
|
+
}
|
|
1870
|
+
function readAuthProfileFlag(parsed) {
|
|
1871
|
+
const profile = asOptionalString(parsed.flags.profile) ?? "default";
|
|
1872
|
+
if (!/^[a-z][a-z0-9-]*$/u.test(profile)) {
|
|
1873
|
+
throw new UsageError("--profile must be a lowercase identifier.");
|
|
1874
|
+
}
|
|
1875
|
+
return profile;
|
|
1876
|
+
}
|
|
1877
|
+
async function resolveAdapterForAuthTarget(dir, targetRaw) {
|
|
1878
|
+
const target = parseWorkbenchAdapterAuthTarget(targetRaw);
|
|
1879
|
+
const spec = (await readLocalProjectSource(dir)).spec;
|
|
1880
|
+
const adapters = await resolveWorkbenchAdaptersForProject(dir, spec);
|
|
1881
|
+
const adapter = adapters.find((entry) => entry.manifest.id === target.adapterId);
|
|
1882
|
+
if (!adapter) {
|
|
1883
|
+
throw new UsageError(`Adapter ${target.adapterId} is not used by this benchmark source. Add it to the benchmark, subject, or optimizer YAML before connecting auth.`);
|
|
1884
|
+
}
|
|
1885
|
+
if (!adapter.manifest.auth) {
|
|
1886
|
+
throw new UsageError(`Adapter ${target.adapterId} does not declare auth.`);
|
|
1887
|
+
}
|
|
1888
|
+
return adapter;
|
|
1889
|
+
}
|
|
1890
|
+
function readAdapterConnectMethod(manifest, slot, parsed) {
|
|
1891
|
+
const supported = adapterAuthMethodNames(manifest, slot);
|
|
1892
|
+
if (supported.length === 0) {
|
|
1893
|
+
const label = slot ? `${manifest.id}/${slot}` : manifest.id;
|
|
1894
|
+
throw new UsageError(`Adapter ${label} does not declare auth methods.`);
|
|
1895
|
+
}
|
|
1896
|
+
const method = asOptionalString(parsed.flags.method) ?? (supported.includes("oauth") ? "oauth" : supported[0]);
|
|
1897
|
+
if (!supported.includes(method)) {
|
|
1898
|
+
const label = slot ? `${manifest.id}/${slot}` : manifest.id;
|
|
1899
|
+
throw new UsageError(`--method ${method} is not supported for ${label}. Supported methods: ${supported.join(", ")}.`);
|
|
1900
|
+
}
|
|
1901
|
+
return method;
|
|
1902
|
+
}
|
|
1903
|
+
function readAdapterAuthMethodEnvEntries(manifest, slot, method) {
|
|
1904
|
+
const methodManifest = adapterAuthMethodManifest(manifest, slot, method);
|
|
1905
|
+
const env = methodManifest?.env;
|
|
1906
|
+
if (!Array.isArray(env) || env.length === 0) {
|
|
1907
|
+
const files = methodManifest?.files;
|
|
1908
|
+
if (Array.isArray(files) && files.length > 0) {
|
|
1909
|
+
return [];
|
|
1910
|
+
}
|
|
1911
|
+
const label = slot ? `${manifest.id}/${slot}` : manifest.id;
|
|
1912
|
+
throw new UsageError(`Adapter ${label} method ${method} cannot be collected by this CLI yet; use an env-backed method.`);
|
|
1913
|
+
}
|
|
1914
|
+
return env.map((entry) => normalizeAdapterAuthEnvManifestEntry(manifest.id, entry));
|
|
1915
|
+
}
|
|
1916
|
+
async function collectAdapterAuthForConnect(args) {
|
|
1917
|
+
const methodManifest = adapterAuthMethodManifest(args.adapter.manifest, args.target.slot, args.method);
|
|
1918
|
+
if (!methodManifest) {
|
|
1919
|
+
throw new UsageError(`Adapter ${args.targetRaw} does not declare method ${args.method}.`);
|
|
1920
|
+
}
|
|
1921
|
+
if (typeof methodManifest.command === "string" && methodManifest.command.trim()) {
|
|
1922
|
+
return adapterAuthBundleFromCommand({
|
|
1923
|
+
target: args.target,
|
|
1924
|
+
method: args.method,
|
|
1925
|
+
command: methodManifest.command.trim(),
|
|
1926
|
+
cwd: args.cwd,
|
|
1927
|
+
});
|
|
1928
|
+
}
|
|
1929
|
+
const envEntries = readAdapterAuthMethodEnvEntries(args.adapter.manifest, args.target.slot, args.method);
|
|
1930
|
+
const env = Object.fromEntries(envEntries.flatMap((entry) => {
|
|
1931
|
+
const value = process.env[entry.name]?.trim() ?? "";
|
|
1932
|
+
return value ? [[entry.name, value]] : [];
|
|
1933
|
+
}));
|
|
1934
|
+
const missing = envEntries
|
|
1935
|
+
.filter((entry) => entry.required && !env[entry.name])
|
|
1936
|
+
.map((entry) => entry.name);
|
|
1937
|
+
if (missing.length > 0) {
|
|
1938
|
+
throw new UsageError(`Missing required auth environment variable${missing.length === 1 ? "" : "s"} for ${args.targetRaw}: ${missing.join(", ")}.`);
|
|
1939
|
+
}
|
|
1940
|
+
const files = await readAdapterAuthMethodFiles({
|
|
1941
|
+
manifest: args.adapter.manifest,
|
|
1942
|
+
slot: args.target.slot,
|
|
1943
|
+
method: args.method,
|
|
1944
|
+
profileRoot: args.profileRoot,
|
|
1945
|
+
});
|
|
1946
|
+
return createWorkbenchAdapterAuthBundle({
|
|
1947
|
+
target: args.target,
|
|
1948
|
+
method: args.method,
|
|
1949
|
+
files,
|
|
1950
|
+
env,
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1953
|
+
async function readAdapterAuthMethodFiles(args) {
|
|
1954
|
+
const methodManifest = adapterAuthMethodManifest(args.manifest, args.slot, args.method);
|
|
1955
|
+
const filePaths = methodManifest?.files;
|
|
1956
|
+
if (!Array.isArray(filePaths) || filePaths.length === 0) {
|
|
1957
|
+
return [];
|
|
1958
|
+
}
|
|
1959
|
+
const files = [];
|
|
1960
|
+
for (const entry of filePaths) {
|
|
1961
|
+
const { path: normalized, required } = normalizeAdapterAuthFileManifestEntry(args.manifest.id, entry);
|
|
1962
|
+
const absolute = path.join(args.profileRoot, normalized);
|
|
1963
|
+
const content = await fs.readFile(absolute, "utf8").catch((error) => {
|
|
1964
|
+
if (error.code === "ENOENT") {
|
|
1965
|
+
if (!required) {
|
|
1966
|
+
return null;
|
|
1967
|
+
}
|
|
1968
|
+
throw new UsageError(`Missing required auth profile file for ${args.manifest.id}: ${absolute}.`);
|
|
1969
|
+
}
|
|
1970
|
+
throw error;
|
|
1971
|
+
});
|
|
1972
|
+
if (content === null) {
|
|
1973
|
+
continue;
|
|
1974
|
+
}
|
|
1975
|
+
files.push({
|
|
1976
|
+
path: normalized,
|
|
1977
|
+
content,
|
|
1978
|
+
encoding: "utf8",
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
return files;
|
|
1982
|
+
}
|
|
1983
|
+
function normalizeAdapterAuthFileManifestEntry(adapterId, entry) {
|
|
1984
|
+
const filePath = typeof entry === "string"
|
|
1985
|
+
? entry
|
|
1986
|
+
: entry && typeof entry === "object" && !Array.isArray(entry)
|
|
1987
|
+
? entry.path
|
|
1988
|
+
: undefined;
|
|
1989
|
+
if (typeof filePath !== "string" || !filePath.trim()) {
|
|
1990
|
+
throw new UsageError(`Adapter ${adapterId} declares invalid auth file path.`);
|
|
1991
|
+
}
|
|
1992
|
+
const normalized = filePath.replace(/\\/gu, "/").replace(/^\/+/u, "");
|
|
1993
|
+
if (!normalized || normalized.split("/").some((part) => part === "." || part === ".." || part === "")) {
|
|
1994
|
+
throw new UsageError(`Adapter ${adapterId} declares unsafe auth file path: ${filePath}.`);
|
|
1995
|
+
}
|
|
1996
|
+
const required = entry && typeof entry === "object" && !Array.isArray(entry)
|
|
1997
|
+
? entry.required !== false
|
|
1998
|
+
: true;
|
|
1999
|
+
return { path: normalized, required };
|
|
2000
|
+
}
|
|
2001
|
+
function normalizeAdapterAuthEnvManifestEntry(adapterId, entry) {
|
|
2002
|
+
const name = typeof entry === "string"
|
|
2003
|
+
? entry
|
|
2004
|
+
: entry && typeof entry === "object" && !Array.isArray(entry)
|
|
2005
|
+
? entry.name
|
|
2006
|
+
: undefined;
|
|
2007
|
+
if (typeof name !== "string" || !/^[A-Z_][A-Z0-9_]*$/u.test(name)) {
|
|
2008
|
+
throw new UsageError(`Adapter ${adapterId} declares invalid auth env var ${String(name)}.`);
|
|
2009
|
+
}
|
|
2010
|
+
const required = entry && typeof entry === "object" && !Array.isArray(entry)
|
|
2011
|
+
? entry.required !== false
|
|
2012
|
+
: true;
|
|
2013
|
+
return { name, required };
|
|
2014
|
+
}
|
|
2015
|
+
async function adapterAuthBundleFromCommand(args) {
|
|
2016
|
+
const { stdout } = await runShellCommand({
|
|
2017
|
+
command: args.command,
|
|
2018
|
+
cwd: args.cwd,
|
|
2019
|
+
env: process.env,
|
|
2020
|
+
});
|
|
2021
|
+
let parsed;
|
|
2022
|
+
try {
|
|
2023
|
+
parsed = JSON.parse(stdout);
|
|
2024
|
+
}
|
|
2025
|
+
catch (error) {
|
|
2026
|
+
throw new UsageError(`Adapter auth command for ${args.target.adapterId} must print one JSON object: ${error instanceof Error ? error.message : String(error)}.`);
|
|
2027
|
+
}
|
|
2028
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
2029
|
+
throw new UsageError(`Adapter auth command for ${args.target.adapterId} must print one JSON object.`);
|
|
2030
|
+
}
|
|
2031
|
+
const record = parsed;
|
|
2032
|
+
const env = normalizeAdapterCommandAuthEnv(record.env);
|
|
2033
|
+
const files = Array.isArray(record.files)
|
|
2034
|
+
? record.files
|
|
2035
|
+
: [];
|
|
2036
|
+
return createWorkbenchAdapterAuthBundle({
|
|
2037
|
+
target: args.target,
|
|
2038
|
+
method: args.method,
|
|
2039
|
+
files,
|
|
2040
|
+
env,
|
|
2041
|
+
});
|
|
2042
|
+
}
|
|
2043
|
+
function normalizeAdapterCommandAuthEnv(value) {
|
|
2044
|
+
if (value === undefined) {
|
|
2045
|
+
return {};
|
|
2046
|
+
}
|
|
2047
|
+
if (Array.isArray(value)) {
|
|
2048
|
+
return Object.fromEntries(value.flatMap((entry) => {
|
|
2049
|
+
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
|
2050
|
+
return [];
|
|
2051
|
+
}
|
|
2052
|
+
const record = entry;
|
|
2053
|
+
return typeof record.name === "string" && typeof record.value === "string"
|
|
2054
|
+
? [[record.name, record.value]]
|
|
2055
|
+
: [];
|
|
2056
|
+
}));
|
|
2057
|
+
}
|
|
2058
|
+
if (value && typeof value === "object") {
|
|
2059
|
+
return Object.fromEntries(Object.entries(value)
|
|
2060
|
+
.filter((entry) => typeof entry[1] === "string"));
|
|
2061
|
+
}
|
|
2062
|
+
throw new UsageError("Adapter auth command env must be an object or a list.");
|
|
2063
|
+
}
|
|
2064
|
+
async function runShellCommand(args) {
|
|
2065
|
+
const label = args.errorLabel ?? args.command;
|
|
2066
|
+
return await new Promise((resolve, reject) => {
|
|
2067
|
+
const child = spawn("sh", ["-c", args.command], {
|
|
2068
|
+
cwd: args.cwd,
|
|
2069
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
2070
|
+
env: args.env,
|
|
2071
|
+
});
|
|
2072
|
+
let stdout = "";
|
|
2073
|
+
let stderr = "";
|
|
2074
|
+
child.stdout?.setEncoding("utf8");
|
|
2075
|
+
child.stderr?.setEncoding("utf8");
|
|
2076
|
+
child.stdout?.on("data", (chunk) => {
|
|
2077
|
+
stdout += chunk;
|
|
2078
|
+
});
|
|
2079
|
+
child.stderr?.on("data", (chunk) => {
|
|
2080
|
+
stderr += chunk;
|
|
2081
|
+
});
|
|
2082
|
+
child.on("error", reject);
|
|
2083
|
+
child.on("exit", (code, signal) => {
|
|
2084
|
+
if (code === 0) {
|
|
2085
|
+
resolve({ stdout, stderr });
|
|
2086
|
+
return;
|
|
2087
|
+
}
|
|
2088
|
+
reject(new Error(`${label} exited with code ${code ?? "null"} signal ${signal ?? "null"}${stderr.trim() ? `: ${stderr.trim()}` : ""}.`));
|
|
2089
|
+
});
|
|
2090
|
+
});
|
|
2091
|
+
}
|
|
2092
|
+
async function readLocalProjectSourceIfPresent(dir) {
|
|
2093
|
+
if (!(await fileIsReadable(path.join(dir, WORKBENCH_BENCHMARK_FILE)))) {
|
|
2094
|
+
return null;
|
|
2095
|
+
}
|
|
2096
|
+
return await readLocalProjectSource(dir);
|
|
2097
|
+
}
|
|
2098
|
+
async function fileIsReadable(filePath) {
|
|
2099
|
+
return await fs.stat(filePath).then((stat) => stat.isFile(), () => false);
|
|
2100
|
+
}
|
|
2101
|
+
function adapterAuthMethodNames(manifest, slot) {
|
|
2102
|
+
const auth = adapterAuthRecord(manifest.auth);
|
|
2103
|
+
if (!auth) {
|
|
2104
|
+
return [];
|
|
2105
|
+
}
|
|
2106
|
+
const methods = slot
|
|
2107
|
+
? adapterAuthRecord(adapterAuthRecord(auth.slots)?.[slot])?.methods
|
|
2108
|
+
: auth.methods;
|
|
2109
|
+
return adapterAuthRecord(methods)
|
|
2110
|
+
? Object.keys(methods).sort()
|
|
2111
|
+
: [];
|
|
2112
|
+
}
|
|
2113
|
+
function adapterAuthMethodManifest(manifest, slot, method) {
|
|
2114
|
+
const auth = adapterAuthRecord(manifest.auth);
|
|
2115
|
+
if (!auth) {
|
|
2116
|
+
return null;
|
|
2117
|
+
}
|
|
2118
|
+
const methods = slot
|
|
2119
|
+
? adapterAuthRecord(adapterAuthRecord(auth.slots)?.[slot])?.methods
|
|
2120
|
+
: auth.methods;
|
|
2121
|
+
const methodManifest = adapterAuthRecord(methods)?.[method];
|
|
2122
|
+
return adapterAuthRecord(methodManifest);
|
|
2123
|
+
}
|
|
2124
|
+
function adapterAuthRecord(value) {
|
|
2125
|
+
return value && typeof value === "object" && !Array.isArray(value)
|
|
2126
|
+
? value
|
|
2127
|
+
: null;
|
|
2128
|
+
}
|
|
2129
|
+
async function pushBenchmark(argv, io) {
|
|
2130
|
+
const parsed = parseArgs(argv);
|
|
2131
|
+
rejectUnknownFlags(parsed, new Set(["dir", "tag", "visibility", "dry-run", "json"]));
|
|
2132
|
+
const dir = resolveSourceDir(parsed);
|
|
2133
|
+
const source = await readLocalProjectSource(dir);
|
|
2134
|
+
const origin = await readWorkbenchOrigin(dir);
|
|
2135
|
+
const baseUrl = await effectiveBaseUrl(origin?.baseUrl);
|
|
2136
|
+
const visibility = readBenchmarkVisibility(parsed.flags.visibility);
|
|
2137
|
+
const dryRun = parsed.flags["dry-run"] === true;
|
|
2138
|
+
if (!origin) {
|
|
2139
|
+
if (dryRun) {
|
|
2140
|
+
writeOutput({
|
|
2141
|
+
ok: true,
|
|
2142
|
+
dryRun: true,
|
|
2143
|
+
action: "create",
|
|
2144
|
+
dir,
|
|
2145
|
+
baseUrl,
|
|
2146
|
+
benchmarkName: source.spec.name,
|
|
2147
|
+
tag: asOptionalString(parsed.flags.tag) ?? null,
|
|
2148
|
+
visibility,
|
|
2149
|
+
sourceFileCount: sourceFileCount(source),
|
|
2150
|
+
}, parsed, io, () => `Would push benchmark ${source.spec.name}.`);
|
|
2151
|
+
return 0;
|
|
2152
|
+
}
|
|
2153
|
+
const response = await apiRequest("/api/workbench/benchmarks", {
|
|
2154
|
+
method: "POST",
|
|
2155
|
+
body: hostedProjectSourceRequest(source),
|
|
2156
|
+
}, baseUrl);
|
|
2157
|
+
const project = response.benchmark;
|
|
2158
|
+
const publishedProject = visibility === "public"
|
|
2159
|
+
? (await apiRequest(projectApiPath(project.id, "/publish"), { method: "PUT" }, baseUrl)).benchmark
|
|
2160
|
+
: project;
|
|
2161
|
+
const nextOrigin = await writeWorkbenchOrigin(dir, {
|
|
2162
|
+
baseUrl,
|
|
2163
|
+
owner: publishedProject.ownerUsername ?? project.ownerUsername ?? "",
|
|
2164
|
+
project: publishedProject.name ?? project.name ?? source.spec.name,
|
|
2165
|
+
projectId: publishedProject.id ?? project.id,
|
|
2166
|
+
writable: true,
|
|
2167
|
+
sourceRevisionId: publishedProject.currentSpecVersionId ?? project.currentSpecVersionId,
|
|
2168
|
+
sourceFingerprint: publishedProject.sourceFingerprint ?? project.sourceFingerprint,
|
|
2169
|
+
});
|
|
2170
|
+
writeOutput({
|
|
2171
|
+
ok: true,
|
|
2172
|
+
action: "create",
|
|
2173
|
+
benchmark: publishedProject,
|
|
2174
|
+
tag: asOptionalString(parsed.flags.tag) ?? null,
|
|
2175
|
+
visibility,
|
|
2176
|
+
origin: nextOrigin,
|
|
2177
|
+
urls: buildWorkbenchResourceUrls({
|
|
2178
|
+
baseUrl,
|
|
2179
|
+
projectId: publishedProject.id ?? project.id,
|
|
2180
|
+
owner: nextOrigin.owner,
|
|
2181
|
+
projectName: nextOrigin.project,
|
|
2182
|
+
}),
|
|
2183
|
+
}, parsed, io, (record) => {
|
|
2184
|
+
const value = record;
|
|
2185
|
+
return [
|
|
2186
|
+
`Pushed ${value.origin.owner}/${value.origin.project} (${value.origin.projectId}).`,
|
|
2187
|
+
`Open benchmark: ${value.urls.benchmark}`,
|
|
2188
|
+
].join("\n");
|
|
2189
|
+
});
|
|
2190
|
+
return 0;
|
|
2191
|
+
}
|
|
2192
|
+
const projectId = origin.projectId;
|
|
2193
|
+
if (!projectId) {
|
|
2194
|
+
throw new UsageError("Missing hosted benchmark. Run workbench push from a source directory.");
|
|
2195
|
+
}
|
|
2196
|
+
if (!origin.writable) {
|
|
2197
|
+
throw new UsageError("Cannot push to a read-only benchmark clone. Run workbench cloud fork to create a writable benchmark fork.");
|
|
2198
|
+
}
|
|
2199
|
+
if (dryRun) {
|
|
2200
|
+
writeOutput({
|
|
2201
|
+
ok: true,
|
|
2202
|
+
dryRun: true,
|
|
2203
|
+
action: "update",
|
|
2204
|
+
dir,
|
|
2205
|
+
baseUrl,
|
|
2206
|
+
benchmarkId: projectId,
|
|
2207
|
+
tag: asOptionalString(parsed.flags.tag) ?? null,
|
|
2208
|
+
visibility,
|
|
2209
|
+
sourceFileCount: sourceFileCount(source),
|
|
2210
|
+
}, parsed, io, () => `Would push ${sourceFileCount(source)} source file(s) to ${projectId}.`);
|
|
2211
|
+
return 0;
|
|
2212
|
+
}
|
|
2213
|
+
const response = await apiRequest(projectApiPath(projectId, "/source"), {
|
|
2214
|
+
method: "PUT",
|
|
2215
|
+
body: hostedProjectSourceRequest(source),
|
|
2216
|
+
}, baseUrl);
|
|
2217
|
+
const publishedProject = visibility === "public"
|
|
2218
|
+
? (await apiRequest(projectApiPath(response.benchmark.id, "/publish"), { method: "PUT" }, baseUrl)).benchmark
|
|
2219
|
+
: response.benchmark;
|
|
2220
|
+
const nextOrigin = await writeWorkbenchOrigin(dir, {
|
|
2221
|
+
baseUrl,
|
|
2222
|
+
owner: publishedProject.ownerUsername ?? response.benchmark.ownerUsername ?? origin.owner,
|
|
2223
|
+
project: publishedProject.name ?? response.benchmark.name ?? origin.project ?? source.spec.name,
|
|
2224
|
+
projectId: publishedProject.id ?? response.benchmark.id,
|
|
2225
|
+
writable: true,
|
|
2226
|
+
sourceRevisionId: publishedProject.currentSpecVersionId ?? response.benchmark.currentSpecVersionId,
|
|
2227
|
+
sourceFingerprint: response.sourceFingerprint ?? publishedProject.sourceFingerprint ?? response.benchmark.sourceFingerprint,
|
|
2228
|
+
upstream: origin.upstream,
|
|
2229
|
+
});
|
|
2230
|
+
writeOutput({
|
|
2231
|
+
ok: true,
|
|
2232
|
+
action: "update",
|
|
2233
|
+
changed: response.changed === true,
|
|
2234
|
+
benchmark: publishedProject,
|
|
2235
|
+
tag: asOptionalString(parsed.flags.tag) ?? null,
|
|
2236
|
+
visibility,
|
|
2237
|
+
origin: nextOrigin,
|
|
2238
|
+
urls: buildWorkbenchResourceUrls({
|
|
2239
|
+
baseUrl,
|
|
2240
|
+
projectId: publishedProject.id ?? response.benchmark.id,
|
|
2241
|
+
owner: nextOrigin.owner,
|
|
2242
|
+
projectName: nextOrigin.project,
|
|
2243
|
+
}),
|
|
2244
|
+
}, parsed, io, (record) => {
|
|
2245
|
+
const value = record;
|
|
2246
|
+
return [
|
|
2247
|
+
`${value.changed ? "Pushed" : "Already up to date"} ${value.origin.owner}/${value.origin.project} (${value.origin.projectId}).`,
|
|
2248
|
+
`Open benchmark: ${value.urls.benchmark}`,
|
|
2249
|
+
].join("\n");
|
|
2250
|
+
});
|
|
2251
|
+
return 0;
|
|
2252
|
+
}
|
|
2253
|
+
function readBenchmarkVisibility(value) {
|
|
2254
|
+
if (value === undefined) {
|
|
2255
|
+
return "public";
|
|
2256
|
+
}
|
|
2257
|
+
if (value === "private" || value === "public") {
|
|
2258
|
+
return value;
|
|
2259
|
+
}
|
|
2260
|
+
throw new UsageError("--visibility must be private or public.");
|
|
2261
|
+
}
|
|
2262
|
+
async function cloneProject(argv, io) {
|
|
2263
|
+
const parsed = parseArgs(argv);
|
|
2264
|
+
rejectUnknownFlags(parsed, new Set(["dry-run", "json"]));
|
|
2265
|
+
const ref = readRequiredBenchmarkRef(parsed);
|
|
2266
|
+
const outputDir = parsed.positionals[1] ?? ref.project;
|
|
2267
|
+
if (parsed.positionals.length > 2) {
|
|
2268
|
+
throw new UsageError("workbench clone accepts OWNER/BENCHMARK[@REF] and an optional output directory.");
|
|
2269
|
+
}
|
|
2270
|
+
const baseUrl = await effectiveBaseUrl();
|
|
2271
|
+
const projectResponse = await apiRequest(publicProjectApiPath(ref), {}, baseUrl);
|
|
2272
|
+
const filesResponse = await apiRequest(publicProjectSourceApiPath(ref), {}, baseUrl);
|
|
2273
|
+
if (parsed.flags["dry-run"] === true) {
|
|
2274
|
+
writeOutput({
|
|
2275
|
+
ok: true,
|
|
2276
|
+
dryRun: true,
|
|
2277
|
+
ref,
|
|
2278
|
+
outputDir,
|
|
2279
|
+
fileCount: filesResponse.files.length,
|
|
2280
|
+
}, parsed, io, () => `Would clone ${formatBenchmarkRef(ref)} to ${outputDir}.`);
|
|
2281
|
+
return 0;
|
|
2282
|
+
}
|
|
2283
|
+
await syncSourceFiles(outputDir, filesResponse.files);
|
|
2284
|
+
const project = projectResponse.benchmark;
|
|
2285
|
+
const sourceProject = filesResponse.benchmark;
|
|
2286
|
+
const origin = await writeWorkbenchOrigin(outputDir, {
|
|
2287
|
+
baseUrl,
|
|
2288
|
+
owner: sourceProject?.ownerUsername ?? project.ownerUsername,
|
|
2289
|
+
project: sourceProject?.name ?? project.name,
|
|
2290
|
+
projectId: sourceProject?.id ?? project.id,
|
|
2291
|
+
writable: false,
|
|
2292
|
+
sourceRevisionId: sourceProject?.currentSpecVersionId ?? project.currentSpecVersionId,
|
|
2293
|
+
sourceFingerprint: sourceProject?.sourceFingerprint ?? project.sourceFingerprint,
|
|
2294
|
+
});
|
|
2295
|
+
writeOutput({
|
|
2296
|
+
ok: true,
|
|
2297
|
+
origin,
|
|
2298
|
+
outputDir,
|
|
2299
|
+
files: filesResponse.files.length,
|
|
2300
|
+
}, parsed, io, (record) => {
|
|
2301
|
+
const value = record;
|
|
2302
|
+
return `Cloned ${value.origin.owner}/${value.origin.project} to ${value.outputDir} (${value.files} file(s)).`;
|
|
2303
|
+
});
|
|
2304
|
+
return 0;
|
|
2305
|
+
}
|
|
2306
|
+
async function pullProject(argv, io) {
|
|
2307
|
+
const parsed = parseArgs(argv);
|
|
2308
|
+
rejectUnknownFlags(parsed, new Set(["dir", "dry-run", "json"]));
|
|
2309
|
+
if (parsed.positionals.length > 0) {
|
|
2310
|
+
throw new UsageError("workbench pull updates the current origin; use workbench clone OWNER/BENCHMARK[@REF] DIR for a new directory.");
|
|
2311
|
+
}
|
|
2312
|
+
const dir = resolveDir(parsed);
|
|
2313
|
+
const origin = await requireWorkbenchOrigin(dir);
|
|
2314
|
+
const filesResponse = origin.writable
|
|
2315
|
+
? await apiRequest(projectApiPath(origin.projectId, "/source"), {}, await effectiveBaseUrl(origin.baseUrl))
|
|
2316
|
+
: await apiRequest(publicProjectSourceApiPath({ owner: origin.owner, project: origin.project }), {}, await effectiveBaseUrl(origin.baseUrl));
|
|
2317
|
+
if (parsed.flags["dry-run"] === true) {
|
|
2318
|
+
writeOutput({
|
|
2319
|
+
ok: true,
|
|
2320
|
+
dryRun: true,
|
|
2321
|
+
dir,
|
|
2322
|
+
fileCount: filesResponse.files.length,
|
|
2323
|
+
}, parsed, io, () => `Would pull ${filesResponse.files.length} source file(s) into ${dir}.`);
|
|
2324
|
+
return 0;
|
|
2325
|
+
}
|
|
2326
|
+
await syncSourceFiles(dir, filesResponse.files);
|
|
2327
|
+
const sourceProject = filesResponse.benchmark;
|
|
2328
|
+
const nextOrigin = await writeWorkbenchOrigin(dir, {
|
|
2329
|
+
...origin,
|
|
2330
|
+
...(sourceProject?.ownerUsername ? { owner: sourceProject.ownerUsername } : {}),
|
|
2331
|
+
...(sourceProject?.name ? { project: sourceProject.name } : {}),
|
|
2332
|
+
...(sourceProject?.id ? { projectId: sourceProject.id } : {}),
|
|
2333
|
+
...(sourceProject?.currentSpecVersionId ? { sourceRevisionId: sourceProject.currentSpecVersionId } : {}),
|
|
2334
|
+
...(sourceProject?.sourceFingerprint ? { sourceFingerprint: sourceProject.sourceFingerprint } : {}),
|
|
2335
|
+
});
|
|
2336
|
+
writeOutput({
|
|
2337
|
+
ok: true,
|
|
2338
|
+
origin: nextOrigin,
|
|
2339
|
+
dir,
|
|
2340
|
+
files: filesResponse.files.length,
|
|
2341
|
+
}, parsed, io, (record) => {
|
|
2342
|
+
const value = record;
|
|
2343
|
+
return `Pulled ${value.files} source file(s) into ${value.dir}.`;
|
|
2344
|
+
});
|
|
2345
|
+
return 0;
|
|
2346
|
+
}
|
|
2347
|
+
async function fetchProject(argv, io) {
|
|
2348
|
+
const parsed = parseArgs(argv);
|
|
2349
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
2350
|
+
if (parsed.positionals.length > 0) {
|
|
2351
|
+
throw new UsageError("workbench fetch updates the current remote cache; use workbench clone OWNER/BENCHMARK[@REF] DIR for a new directory.");
|
|
2352
|
+
}
|
|
2353
|
+
const dir = resolveDir(parsed);
|
|
2354
|
+
const origin = await requireWorkbenchOrigin(dir);
|
|
2355
|
+
const filesResponse = await readRemoteSourceFiles(origin);
|
|
2356
|
+
const fetchRoot = path.join(dir, ".workbench", "fetch");
|
|
2357
|
+
await fs.rm(fetchRoot, { force: true, recursive: true });
|
|
2358
|
+
await fs.mkdir(fetchRoot, { recursive: true });
|
|
2359
|
+
await writeFiles(path.join(fetchRoot, "source"), filesResponse.files);
|
|
2360
|
+
const sourceProject = filesResponse.benchmark;
|
|
2361
|
+
const nextOrigin = await writeWorkbenchOrigin(dir, {
|
|
2362
|
+
...origin,
|
|
2363
|
+
...(sourceProject?.ownerUsername ? { owner: sourceProject.ownerUsername } : {}),
|
|
2364
|
+
...(sourceProject?.name ? { project: sourceProject.name } : {}),
|
|
2365
|
+
...(sourceProject?.id ? { projectId: sourceProject.id } : {}),
|
|
2366
|
+
...(sourceProject?.currentSpecVersionId ? { sourceRevisionId: sourceProject.currentSpecVersionId } : {}),
|
|
2367
|
+
...(sourceProject?.sourceFingerprint ? { sourceFingerprint: sourceProject.sourceFingerprint } : {}),
|
|
2368
|
+
});
|
|
2369
|
+
await fs.writeFile(path.join(fetchRoot, "manifest.json"), `${JSON.stringify({
|
|
2370
|
+
fetchedAt: new Date().toISOString(),
|
|
2371
|
+
origin: nextOrigin,
|
|
2372
|
+
files: filesResponse.files.map((file) => file.path),
|
|
2373
|
+
}, null, 2)}\n`);
|
|
2374
|
+
writeOutput({
|
|
2375
|
+
ok: true,
|
|
2376
|
+
origin: nextOrigin,
|
|
2377
|
+
dir,
|
|
2378
|
+
fetchRoot,
|
|
2379
|
+
files: filesResponse.files.length,
|
|
2380
|
+
}, parsed, io, (record) => {
|
|
2381
|
+
const value = record;
|
|
2382
|
+
return `Fetched ${value.files} source file(s) into ${value.fetchRoot}.`;
|
|
2383
|
+
});
|
|
2384
|
+
return 0;
|
|
2385
|
+
}
|
|
2386
|
+
async function readRemoteSourceFiles(origin) {
|
|
2387
|
+
return origin.writable
|
|
2388
|
+
? await apiRequest(projectApiPath(origin.projectId, "/source"), {}, await effectiveBaseUrl(origin.baseUrl))
|
|
2389
|
+
: await apiRequest(publicProjectSourceApiPath({ owner: origin.owner, project: origin.project }), {}, await effectiveBaseUrl(origin.baseUrl));
|
|
2390
|
+
}
|
|
2391
|
+
async function runRemoteCommand(argv, io) {
|
|
2392
|
+
const command = argv[0] ?? "show";
|
|
2393
|
+
switch (command) {
|
|
2394
|
+
case "show":
|
|
2395
|
+
return await remoteShow(argv.slice(1), io);
|
|
2396
|
+
case "add":
|
|
2397
|
+
return await remoteAdd(argv.slice(1), io, "add");
|
|
2398
|
+
case "set-url":
|
|
2399
|
+
return await remoteAdd(argv.slice(1), io, "set-url");
|
|
2400
|
+
case "remove":
|
|
2401
|
+
return await remoteRemove(argv.slice(1), io);
|
|
2402
|
+
default:
|
|
2403
|
+
throw new UsageError(`Unknown command: remote ${argv.join(" ")}`);
|
|
2404
|
+
}
|
|
2405
|
+
}
|
|
2406
|
+
async function remoteShow(argv, io) {
|
|
2407
|
+
const parsed = parseArgs(argv);
|
|
2408
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
2409
|
+
const origin = await requireWorkbenchOrigin(resolveDir(parsed));
|
|
2410
|
+
writeOutput({ ok: true, remote: "origin", origin }, parsed, io, (record) => {
|
|
2411
|
+
const value = record;
|
|
2412
|
+
return [
|
|
2413
|
+
`origin\t${value.origin.owner}/${value.origin.project}`,
|
|
2414
|
+
`url\t${value.origin.baseUrl}`,
|
|
2415
|
+
`writable\t${value.origin.writable ? "yes" : "no"}`,
|
|
2416
|
+
...(value.origin.sourceFingerprint ? [`fingerprint\t${value.origin.sourceFingerprint}`] : []),
|
|
2417
|
+
].join("\n");
|
|
2418
|
+
});
|
|
2419
|
+
return 0;
|
|
2420
|
+
}
|
|
2421
|
+
async function remoteAdd(argv, io, command) {
|
|
2422
|
+
const parsed = parseArgs(argv);
|
|
2423
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
2424
|
+
const [name, refValue] = parsed.positionals;
|
|
2425
|
+
if (name !== "origin" || !refValue || parsed.positionals.length !== 2) {
|
|
2426
|
+
throw new UsageError(`workbench remote ${command} accepts: origin OWNER/BENCHMARK[@REF].`);
|
|
2427
|
+
}
|
|
2428
|
+
const ref = parseBenchmarkRef(refValue);
|
|
2429
|
+
const baseUrl = await effectiveBaseUrl();
|
|
2430
|
+
const project = await resolveRemoteProject(formatBenchmarkRef(ref), baseUrl);
|
|
2431
|
+
const origin = await writeWorkbenchOrigin(resolveDir(parsed), {
|
|
2432
|
+
baseUrl,
|
|
2433
|
+
owner: project.ownerUsername ?? ref.owner,
|
|
2434
|
+
project: project.name ?? ref.project,
|
|
2435
|
+
projectId: project.id,
|
|
2436
|
+
writable: false,
|
|
2437
|
+
...(project.currentSpecVersionId ? { sourceRevisionId: project.currentSpecVersionId } : {}),
|
|
2438
|
+
...(project.sourceFingerprint ? { sourceFingerprint: project.sourceFingerprint } : {}),
|
|
2439
|
+
});
|
|
2440
|
+
writeOutput({ ok: true, remote: "origin", origin }, parsed, io, () => `Set origin to ${origin.owner}/${origin.project}.`);
|
|
2441
|
+
return 0;
|
|
2442
|
+
}
|
|
2443
|
+
async function remoteRemove(argv, io) {
|
|
2444
|
+
const parsed = parseArgs(argv);
|
|
2445
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
2446
|
+
const [name] = parsed.positionals;
|
|
2447
|
+
if (name !== "origin" || parsed.positionals.length !== 1) {
|
|
2448
|
+
throw new UsageError("workbench remote remove accepts: origin.");
|
|
2449
|
+
}
|
|
2450
|
+
const originPath = workbenchOriginPath(resolveDir(parsed));
|
|
2451
|
+
await fs.rm(originPath, { force: true });
|
|
2452
|
+
writeOutput({ ok: true, remote: "origin", removed: originPath }, parsed, io, () => `Removed origin (${originPath}).`);
|
|
2453
|
+
return 0;
|
|
2454
|
+
}
|
|
2455
|
+
async function forkProject(argv, io) {
|
|
2456
|
+
const parsed = parseArgs(argv);
|
|
2457
|
+
rejectUnknownFlags(parsed, new Set(["json"]));
|
|
2458
|
+
const ref = readRequiredBenchmarkRef(parsed);
|
|
2459
|
+
if (parsed.positionals.length > 2) {
|
|
2460
|
+
throw new UsageError("workbench cloud fork accepts OWNER/BENCHMARK[@REF] and an optional fork name.");
|
|
2461
|
+
}
|
|
2462
|
+
const baseUrl = await effectiveBaseUrl();
|
|
2463
|
+
const response = await apiRequest(`${publicProjectApiPath(ref)}/fork`, {
|
|
2464
|
+
method: "POST",
|
|
2465
|
+
body: { name: parsed.positionals[1] },
|
|
2466
|
+
}, baseUrl);
|
|
2467
|
+
const benchmark = response.benchmark;
|
|
2468
|
+
const currentDir = resolveDir(parsed);
|
|
2469
|
+
const currentOrigin = await readWorkbenchOrigin(currentDir);
|
|
2470
|
+
const outputDir = parsed.positionals[1] ?? (currentOrigin &&
|
|
2471
|
+
!currentOrigin.writable &&
|
|
2472
|
+
currentOrigin.owner === ref.owner &&
|
|
2473
|
+
currentOrigin.project === ref.project
|
|
2474
|
+
? currentDir
|
|
2475
|
+
: benchmark.name);
|
|
2476
|
+
const filesResponse = await apiRequest(projectApiPath(benchmark.id, "/source"), {}, baseUrl);
|
|
2477
|
+
await syncSourceFiles(outputDir, filesResponse.files);
|
|
2478
|
+
const origin = await writeWorkbenchOrigin(outputDir, {
|
|
2479
|
+
baseUrl,
|
|
2480
|
+
owner: benchmark.ownerUsername,
|
|
2481
|
+
project: benchmark.name,
|
|
2482
|
+
projectId: benchmark.id,
|
|
2483
|
+
writable: true,
|
|
2484
|
+
sourceRevisionId: filesResponse.benchmark?.currentSpecVersionId ?? benchmark.currentSpecVersionId,
|
|
2485
|
+
sourceFingerprint: filesResponse.benchmark?.sourceFingerprint ?? benchmark.sourceFingerprint,
|
|
2486
|
+
upstream: originUpstreamFromForkedFrom(benchmark.forkedFrom),
|
|
2487
|
+
});
|
|
2488
|
+
const urls = buildWorkbenchResourceUrls({
|
|
2489
|
+
baseUrl,
|
|
2490
|
+
projectId: benchmark.id,
|
|
2491
|
+
owner: benchmark.ownerUsername,
|
|
2492
|
+
projectName: benchmark.name,
|
|
2493
|
+
});
|
|
2494
|
+
writeOutput({
|
|
2495
|
+
ok: true,
|
|
2496
|
+
benchmark,
|
|
2497
|
+
origin,
|
|
2498
|
+
outputDir,
|
|
2499
|
+
files: filesResponse.files.length,
|
|
2500
|
+
urls,
|
|
2501
|
+
}, parsed, io, (record) => {
|
|
2502
|
+
const value = record;
|
|
2503
|
+
return [
|
|
2504
|
+
`Forked ${formatBenchmarkRef(ref)} to ${value.benchmark.ownerUsername}/${value.benchmark.name} (${value.benchmark.id}).`,
|
|
2505
|
+
`Local checkout: ${value.outputDir} (${value.files} file(s)).`,
|
|
2506
|
+
`Open benchmark: ${value.urls.benchmark}`,
|
|
2507
|
+
].join("\n");
|
|
2508
|
+
});
|
|
2509
|
+
return 0;
|
|
2510
|
+
}
|
|
2511
|
+
async function starProject(argv, io, starred) {
|
|
2512
|
+
const parsed = parseArgs(argv);
|
|
2513
|
+
rejectUnknownFlags(parsed, new Set(["json"]));
|
|
2514
|
+
const ref = readRequiredBenchmarkRef(parsed);
|
|
2515
|
+
if (parsed.positionals.length > 1) {
|
|
2516
|
+
throw new UsageError(`${starred ? "workbench cloud star" : "workbench cloud unstar"} accepts exactly one OWNER/BENCHMARK ref.`);
|
|
2517
|
+
}
|
|
2518
|
+
const response = await apiRequest(`${publicProjectApiPath(ref)}/star`, { method: starred ? "PUT" : "DELETE" }, await effectiveBaseUrl());
|
|
2519
|
+
writeOutput({ ok: true, benchmark: response.benchmark }, parsed, io, (record) => {
|
|
2520
|
+
const value = record;
|
|
2521
|
+
return `${starred ? "Starred" : "Unstarred"} ${formatBenchmarkRef(ref)}; ${value.benchmark.starCount} star(s).`;
|
|
2522
|
+
});
|
|
2523
|
+
return 0;
|
|
2524
|
+
}
|
|
2525
|
+
function originUpstreamFromForkedFrom(forkedFrom) {
|
|
2526
|
+
if (!forkedFrom?.projectId ||
|
|
2527
|
+
!forkedFrom.ownerUsername ||
|
|
2528
|
+
!forkedFrom.benchmarkName ||
|
|
2529
|
+
!forkedFrom.sourceRevisionId) {
|
|
2530
|
+
return undefined;
|
|
2531
|
+
}
|
|
2532
|
+
return {
|
|
2533
|
+
owner: forkedFrom.ownerUsername,
|
|
2534
|
+
project: forkedFrom.benchmarkName,
|
|
2535
|
+
projectId: forkedFrom.projectId,
|
|
2536
|
+
sourceRevisionId: forkedFrom.sourceRevisionId,
|
|
2537
|
+
};
|
|
2538
|
+
}
|
|
2539
|
+
async function startHostedWorkflow(workflow, argv, io) {
|
|
2540
|
+
const parsed = parseArgs(argv);
|
|
2541
|
+
rejectUnknownFlags(parsed, new Set([
|
|
2542
|
+
"dir",
|
|
2543
|
+
"benchmark",
|
|
2544
|
+
"base",
|
|
2545
|
+
"optimizer",
|
|
2546
|
+
"budget",
|
|
2547
|
+
"samples",
|
|
2548
|
+
"watch",
|
|
2549
|
+
"dry-run",
|
|
2550
|
+
"interval-ms",
|
|
2551
|
+
"timeout-ms",
|
|
2552
|
+
"json",
|
|
2553
|
+
]));
|
|
2554
|
+
if (parsed.positionals.length > 1) {
|
|
2555
|
+
throw new UsageError(`workbench cloud ${workflow} accepts at most one source file or directory argument.`);
|
|
2556
|
+
}
|
|
2557
|
+
const optimizerPath = asOptionalString(parsed.flags.optimizer);
|
|
2558
|
+
const sourceArg = parsed.positionals[0] ?? asOptionalString(parsed.flags.dir) ?? process.cwd();
|
|
2559
|
+
if (parsed.positionals.length > 0 && parsed.flags.dir !== undefined) {
|
|
2560
|
+
throw new UsageError("Use either --dir or SOURCE, not both.");
|
|
2561
|
+
}
|
|
2562
|
+
const baseSubjectId = asOptionalString(parsed.flags.base);
|
|
2563
|
+
const request = workflow === "improve"
|
|
2564
|
+
? {
|
|
2565
|
+
workflow,
|
|
2566
|
+
budget: parsePositiveInt(parsed.flags.budget, 1, "budget"),
|
|
2567
|
+
samples: parsePositiveInt(parsed.flags.samples, 1, "samples"),
|
|
2568
|
+
...(baseSubjectId ? { subjectId: baseSubjectId } : {}),
|
|
2569
|
+
}
|
|
2570
|
+
: {
|
|
2571
|
+
workflow,
|
|
2572
|
+
samples: parsePositiveInt(parsed.flags.samples, 1, "samples"),
|
|
2573
|
+
...(baseSubjectId ? { subjectId: baseSubjectId } : {}),
|
|
2574
|
+
};
|
|
2575
|
+
if (workflow === "improve" && !optimizerPath) {
|
|
2576
|
+
throw new UsageError("workbench cloud improve requires --optimizer OPTIMIZER_YAML.");
|
|
2577
|
+
}
|
|
2578
|
+
if (parsed.flags.watch !== true && (parsed.flags["interval-ms"] !== undefined ||
|
|
2579
|
+
parsed.flags["timeout-ms"] !== undefined)) {
|
|
2580
|
+
throw new UsageError("--interval-ms and --timeout-ms require --watch.");
|
|
2581
|
+
}
|
|
2582
|
+
const projectSource = await readLocalProjectSource(path.resolve(sourceArg), {
|
|
2583
|
+
optimizerPath,
|
|
2584
|
+
});
|
|
2585
|
+
if (workflow === "eval") {
|
|
2586
|
+
request.subjectSource = projectSource.subjectSource;
|
|
2587
|
+
request.subjectFiles = projectSource.subjectFiles;
|
|
2588
|
+
request.adapterFiles = projectSource.adapterFiles;
|
|
2589
|
+
}
|
|
2590
|
+
if (workflow === "improve" && projectSource.optimizerSource) {
|
|
2591
|
+
request.optimizerSource = projectSource.optimizerSource;
|
|
2592
|
+
request.adapterFiles = projectSource.adapterFiles;
|
|
2593
|
+
}
|
|
2594
|
+
const watchIntervalMs = parsed.flags.watch === true
|
|
2595
|
+
? parsePositiveInt(parsed.flags["interval-ms"], 1000, "interval-ms")
|
|
2596
|
+
: undefined;
|
|
2597
|
+
const watchTimeoutMs = parsed.flags.watch === true
|
|
2598
|
+
? parseOptionalPositiveInt(parsed.flags["timeout-ms"], "timeout-ms")
|
|
2599
|
+
: undefined;
|
|
2600
|
+
const target = await resolveHostedTarget(parsed, {
|
|
2601
|
+
requireProjectIdentity: true,
|
|
2602
|
+
sourceDir: projectSource.dir,
|
|
2603
|
+
});
|
|
2604
|
+
const dryRun = parsed.flags["dry-run"] === true;
|
|
2605
|
+
if (workflow === "improve" && !dryRun) {
|
|
2606
|
+
request.subjectId = await ensureHostedImproveBaseSubject({
|
|
2607
|
+
parsed,
|
|
2608
|
+
target,
|
|
2609
|
+
samples: request.samples,
|
|
2610
|
+
subjectId: baseSubjectId,
|
|
2611
|
+
intervalMs: watchIntervalMs ?? 1000,
|
|
2612
|
+
timeoutMs: watchTimeoutMs,
|
|
2613
|
+
});
|
|
2614
|
+
}
|
|
2615
|
+
if (dryRun) {
|
|
2616
|
+
writeOutput({
|
|
2617
|
+
ok: true,
|
|
2618
|
+
dryRun: true,
|
|
2619
|
+
projectId: target.projectId,
|
|
2620
|
+
dir: target.dir,
|
|
2621
|
+
baseUrl: target.baseUrl,
|
|
2622
|
+
request,
|
|
2623
|
+
}, parsed, io, () => `Would start hosted ${workflow} for ${target.projectId}.`);
|
|
2624
|
+
return 0;
|
|
2625
|
+
}
|
|
2626
|
+
const response = await apiRequest(projectApiPath(target.projectId, "/runs"), {
|
|
2627
|
+
method: "POST",
|
|
2628
|
+
body: request,
|
|
2629
|
+
}, target.baseUrl);
|
|
2630
|
+
const startedRun = withRunUrls(target, response.run);
|
|
2631
|
+
if (parsed.flags.watch === true) {
|
|
2632
|
+
if (parsed.flags.json !== true) {
|
|
2633
|
+
io.stdout.write(`${formatHostedRunStarted(startedRun, workflow).trimEnd()}\n${HOSTED_WATCH_LIFECYCLE_NOTE}\n`);
|
|
2634
|
+
}
|
|
2635
|
+
const watched = await watchHostedRun({
|
|
2636
|
+
parsed,
|
|
2637
|
+
target,
|
|
2638
|
+
runId: response.run.id,
|
|
2639
|
+
intervalMs: watchIntervalMs ?? 1000,
|
|
2640
|
+
timeoutMs: watchTimeoutMs,
|
|
2641
|
+
});
|
|
2642
|
+
const outputRun = await withHostedRunFailureSummary(target, watched);
|
|
2643
|
+
writeOutput(withRunUrls(target, outputRun), parsed, io, formatHostedRunResult);
|
|
2644
|
+
return hostedRunSucceeded(watched) ? 0 : 1;
|
|
2645
|
+
}
|
|
2646
|
+
writeOutput(startedRun, parsed, io, (run) => formatHostedRunStarted(run, workflow).trimEnd());
|
|
2647
|
+
return 0;
|
|
2648
|
+
}
|
|
2649
|
+
async function ensureHostedImproveBaseSubject(args) {
|
|
2650
|
+
if (args.subjectId) {
|
|
2651
|
+
const response = await apiRequest(projectApiPath(args.target.projectId, "/subjects"), {}, args.target.baseUrl);
|
|
2652
|
+
const subject = response.subjects.find((entry) => entry.id === args.subjectId);
|
|
2653
|
+
if (!subject) {
|
|
2654
|
+
throw new UsageError(`Base subject ${args.subjectId} was not found for the current benchmark.`);
|
|
2655
|
+
}
|
|
2656
|
+
if (subject && (subject.status === "evaluated" || subject.eval != null)) {
|
|
2657
|
+
return args.subjectId;
|
|
2658
|
+
}
|
|
2659
|
+
}
|
|
2660
|
+
const response = await apiRequest(projectApiPath(args.target.projectId, "/runs"), {
|
|
2661
|
+
method: "POST",
|
|
2662
|
+
body: {
|
|
2663
|
+
workflow: "eval",
|
|
2664
|
+
samples: args.samples,
|
|
2665
|
+
...(args.subjectId ? { subjectId: args.subjectId } : {}),
|
|
2666
|
+
},
|
|
2667
|
+
}, args.target.baseUrl);
|
|
2668
|
+
const watched = await watchHostedRun({
|
|
2669
|
+
parsed: args.parsed,
|
|
2670
|
+
target: args.target,
|
|
2671
|
+
runId: response.run.id,
|
|
2672
|
+
intervalMs: args.intervalMs,
|
|
2673
|
+
timeoutMs: args.timeoutMs,
|
|
2674
|
+
});
|
|
2675
|
+
if (!hostedRunSucceeded(watched)) {
|
|
2676
|
+
throw new UsageError(`Parent subject eval ${watched.id} failed; improve was not started.`);
|
|
2677
|
+
}
|
|
2678
|
+
if (!watched.subjectId) {
|
|
2679
|
+
throw new UsageError(`Parent subject eval ${watched.id} did not produce a subject.`);
|
|
2680
|
+
}
|
|
2681
|
+
return watched.subjectId;
|
|
2682
|
+
}
|
|
2683
|
+
async function benchmarkList(argv, io) {
|
|
2684
|
+
const parsed = parseArgs(argv);
|
|
2685
|
+
rejectUnknownFlags(parsed, new Set(["json"]));
|
|
2686
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud benchmarks list", 0);
|
|
2687
|
+
const response = await apiRequest("/api/workbench/public/benchmarks");
|
|
2688
|
+
writeOutput(response.benchmarks, parsed, io, (projects) => {
|
|
2689
|
+
if (projects.length === 0) {
|
|
2690
|
+
return "No hosted Workbench benchmarks.";
|
|
2691
|
+
}
|
|
2692
|
+
return projects
|
|
2693
|
+
.map((project) => `${project.id}\t${project.name}\t${project.runCount} runs\t${project.subjectCount} subjects`)
|
|
2694
|
+
.join("\n");
|
|
2695
|
+
});
|
|
2696
|
+
return 0;
|
|
2697
|
+
}
|
|
2698
|
+
async function benchmarkShow(argv, io) {
|
|
2699
|
+
const parsed = parseArgs(argv);
|
|
2700
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
2701
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud benchmarks show", 1);
|
|
2702
|
+
const dir = resolveDir(parsed);
|
|
2703
|
+
const origin = await readWorkbenchOrigin(dir);
|
|
2704
|
+
const projectRef = parsed.positionals[0] ??
|
|
2705
|
+
origin?.projectId;
|
|
2706
|
+
if (!projectRef) {
|
|
2707
|
+
throw new UsageError("Missing hosted benchmark. Pass OWNER/BENCHMARK, run workbench push, or run workbench clone.");
|
|
2708
|
+
}
|
|
2709
|
+
const response = await apiRequest(benchmarkApiPath(projectRef), {}, await effectiveBaseUrl(origin?.baseUrl));
|
|
2710
|
+
writeOutput(response.benchmark, parsed, io, (project) => {
|
|
2711
|
+
const record = project;
|
|
2712
|
+
return `${record.name} (${record.id})\n${record.runs.length} runs\n${record.subjects.length} subjects`;
|
|
2713
|
+
});
|
|
2714
|
+
return 0;
|
|
2715
|
+
}
|
|
2716
|
+
async function benchmarkDelete(argv, io) {
|
|
2717
|
+
const parsed = parseArgs(argv);
|
|
2718
|
+
rejectUnknownFlags(parsed, new Set(["dir", "dry-run", "json"]));
|
|
2719
|
+
if (parsed.positionals.length > 1) {
|
|
2720
|
+
throw new UsageError(`Unexpected argument for workbench benchmarks delete: ${parsed.positionals.slice(1).join(" ")}`);
|
|
2721
|
+
}
|
|
2722
|
+
const dir = resolveDir(parsed);
|
|
2723
|
+
const origin = await readWorkbenchOrigin(dir);
|
|
2724
|
+
const projectRef = parsed.positionals[0] ??
|
|
2725
|
+
origin?.projectId;
|
|
2726
|
+
if (!projectRef) {
|
|
2727
|
+
throw new UsageError("Missing hosted benchmark. Pass OWNER/BENCHMARK, run workbench push, or run workbench clone.");
|
|
2728
|
+
}
|
|
2729
|
+
const originPath = workbenchOriginPath(dir);
|
|
2730
|
+
const baseUrl = await effectiveBaseUrl(origin?.baseUrl);
|
|
2731
|
+
const project = await resolveRemoteProject(projectRef, baseUrl);
|
|
2732
|
+
const projectId = project.id;
|
|
2733
|
+
const projectName = project.name;
|
|
2734
|
+
const originProjectDeleted = origin ? origin.projectId === projectId : false;
|
|
2735
|
+
if (parsed.flags["dry-run"] === true) {
|
|
2736
|
+
writeOutput({
|
|
2737
|
+
ok: true,
|
|
2738
|
+
dryRun: true,
|
|
2739
|
+
projectId,
|
|
2740
|
+
...(projectName ? { projectName } : {}),
|
|
2741
|
+
baseUrl,
|
|
2742
|
+
...(originProjectDeleted ? { originPath } : {}),
|
|
2743
|
+
}, parsed, io, () => originProjectDeleted
|
|
2744
|
+
? `Would delete hosted benchmark ${formatProjectRef(project)} and remove local origin ${originPath}.`
|
|
2745
|
+
: `Would delete hosted benchmark ${formatProjectRef(project)}.`);
|
|
2746
|
+
return 0;
|
|
2747
|
+
}
|
|
2748
|
+
await apiRequest(projectApiPath(projectId), { method: "DELETE" }, baseUrl);
|
|
2749
|
+
if (originProjectDeleted) {
|
|
2750
|
+
await fs.rm(originPath, { force: true });
|
|
2751
|
+
}
|
|
2752
|
+
writeOutput({
|
|
2753
|
+
ok: true,
|
|
2754
|
+
deleted: true,
|
|
2755
|
+
projectId,
|
|
2756
|
+
...(projectName ? { projectName } : {}),
|
|
2757
|
+
originRemoved: originProjectDeleted,
|
|
2758
|
+
...(originProjectDeleted ? { originPath } : {}),
|
|
2759
|
+
}, parsed, io, () => originProjectDeleted
|
|
2760
|
+
? `Deleted benchmark ${formatProjectRef(project)} and removed local origin ${originPath}.`
|
|
2761
|
+
: `Deleted benchmark ${formatProjectRef(project)}.`);
|
|
2762
|
+
return 0;
|
|
2763
|
+
}
|
|
2764
|
+
async function benchmarkVersions(argv, io) {
|
|
2765
|
+
const parsed = parseArgs(argv);
|
|
2766
|
+
rejectUnknownFlags(parsed, new Set(["dir", "json"]));
|
|
2767
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud benchmarks versions", 1);
|
|
2768
|
+
const projectRef = parsed.positionals[0];
|
|
2769
|
+
const origin = await readWorkbenchOrigin(resolveDir(parsed));
|
|
2770
|
+
if (!projectRef && !origin) {
|
|
2771
|
+
throw new UsageError("Missing benchmark ref. Pass OWNER/BENCHMARK or run from a benchmark clone.");
|
|
2772
|
+
}
|
|
2773
|
+
const response = await apiRequest(benchmarkApiPath(projectRef ?? origin.projectId), {}, await effectiveBaseUrl(origin?.baseUrl));
|
|
2774
|
+
const version = response.benchmark.sourceFingerprint ?? response.benchmark.currentSpecVersionId ?? "current";
|
|
2775
|
+
writeOutput({
|
|
2776
|
+
ok: true,
|
|
2777
|
+
benchmark: response.benchmark,
|
|
2778
|
+
versions: [{ ref: "main", digest: version, current: true }],
|
|
2779
|
+
}, parsed, io, () => `${response.benchmark.name ?? projectRef ?? origin.project}\tmain\t${shortDigest(version)}\tcurrent`);
|
|
2780
|
+
return 0;
|
|
2781
|
+
}
|
|
2782
|
+
async function benchmarkStarred(argv, io) {
|
|
2783
|
+
const parsed = parseArgs(argv);
|
|
2784
|
+
rejectUnknownFlags(parsed, new Set(["json"]));
|
|
2785
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud benchmarks starred", 0);
|
|
2786
|
+
const response = await apiRequest("/api/workbench/benchmarks");
|
|
2787
|
+
const starred = response.benchmarks.filter((project) => project.viewerHasStarred === true);
|
|
2788
|
+
writeOutput(starred, parsed, io, (benchmarks) => {
|
|
2789
|
+
if (benchmarks.length === 0) {
|
|
2790
|
+
return "No starred benchmarks.";
|
|
2791
|
+
}
|
|
2792
|
+
return benchmarks
|
|
2793
|
+
.map((benchmark) => `${benchmark.ownerUsername ?? "-"} / ${benchmark.name ?? "-"}\t${benchmark.starCount ?? 0} stars`)
|
|
2794
|
+
.join("\n");
|
|
2795
|
+
});
|
|
2796
|
+
return 0;
|
|
2797
|
+
}
|
|
2798
|
+
async function subjectList(argv, io) {
|
|
2799
|
+
const parsed = parseArgs(argv);
|
|
2800
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "json"]));
|
|
2801
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud subjects list", 0);
|
|
2802
|
+
const target = await resolveHostedTarget(parsed);
|
|
2803
|
+
const response = await apiRequest(projectApiPath(target.projectId, "/subjects"), {}, target.baseUrl);
|
|
2804
|
+
writeOutput(response.subjects, parsed, io, (subjects) => {
|
|
2805
|
+
if (subjects.length === 0) {
|
|
2806
|
+
return "No subjects yet.";
|
|
2807
|
+
}
|
|
2808
|
+
return subjects
|
|
2809
|
+
.map((subject) => `${subject.id}\t${subject.status}\tmetrics ${formatMetricSummary(subject.metrics)}\t${subject.fileChanges?.length ?? 0} files`)
|
|
2810
|
+
.join("\n");
|
|
2811
|
+
});
|
|
2812
|
+
return 0;
|
|
2813
|
+
}
|
|
2814
|
+
async function subjectShow(argv, io) {
|
|
2815
|
+
const parsed = parseArgs(argv);
|
|
2816
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "json"]));
|
|
2817
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud subjects show", 1);
|
|
2818
|
+
const target = await resolveHostedTarget(parsed);
|
|
2819
|
+
const subjectId = readRequiredSubjectId(parsed);
|
|
2820
|
+
const params = new URLSearchParams({ id: subjectId });
|
|
2821
|
+
const subject = await apiRequest(projectApiPath(target.projectId, `/workbench/record?${params.toString()}`), {}, target.baseUrl);
|
|
2822
|
+
writeOutput(subject, parsed, io, (record) => {
|
|
2823
|
+
const value = record;
|
|
2824
|
+
return [
|
|
2825
|
+
`${value.id ?? subjectId}\t${value.status ?? "unknown"}`,
|
|
2826
|
+
...(value.benchmarkFingerprint ? [`Benchmark version: ${shortDigest(value.benchmarkFingerprint)}`] : []),
|
|
2827
|
+
...(value.subjectFingerprint ? [`Subject digest: ${shortDigest(value.subjectFingerprint)}`] : []),
|
|
2828
|
+
].join("\n");
|
|
2829
|
+
});
|
|
2830
|
+
return 0;
|
|
2831
|
+
}
|
|
2832
|
+
async function subjectFiles(argv, io) {
|
|
2833
|
+
const parsed = parseArgs(argv);
|
|
2834
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "json"]));
|
|
2835
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud subjects files", 1);
|
|
2836
|
+
const target = await resolveHostedTarget(parsed);
|
|
2837
|
+
const subjectId = readRequiredSubjectId(parsed);
|
|
2838
|
+
const response = await apiRequest(projectApiPath(target.projectId, `/subjects/${encodeURIComponent(subjectId)}/files`), {}, target.baseUrl);
|
|
2839
|
+
writeOutput(response.files, parsed, io, (files) => files
|
|
2840
|
+
.map((file) => `${file.path}\t${file.status}\t${file.preview_kind}`)
|
|
2841
|
+
.join("\n") || "No files.");
|
|
2842
|
+
return 0;
|
|
2843
|
+
}
|
|
2844
|
+
async function subjectPreview(argv, io) {
|
|
2845
|
+
const parsed = parseArgs(argv);
|
|
2846
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "path", "output", "json"]));
|
|
2847
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud subjects preview", 1);
|
|
2848
|
+
const target = await resolveHostedTarget(parsed);
|
|
2849
|
+
const subjectId = readRequiredSubjectId(parsed);
|
|
2850
|
+
const filePath = requireFlag(parsed, "path");
|
|
2851
|
+
const params = new URLSearchParams({ path: filePath });
|
|
2852
|
+
const response = await apiRequest(projectApiPath(target.projectId, `/subjects/${encodeURIComponent(subjectId)}/files?${params.toString()}`), {}, target.baseUrl);
|
|
2853
|
+
const content = response.preview.source?.content ??
|
|
2854
|
+
response.preview.rendered_html ??
|
|
2855
|
+
response.preview.diff ??
|
|
2856
|
+
"";
|
|
2857
|
+
const outputPath = asOptionalString(parsed.flags.output);
|
|
2858
|
+
if (outputPath && outputPath !== "-") {
|
|
2859
|
+
await fs.writeFile(outputPath, content);
|
|
2860
|
+
io.stdout.write(`Wrote preview to ${outputPath}\n`);
|
|
2861
|
+
}
|
|
2862
|
+
else if (parsed.flags.json === true) {
|
|
2863
|
+
writeJson(response.preview, io);
|
|
2864
|
+
}
|
|
2865
|
+
else {
|
|
2866
|
+
io.stdout.write(content);
|
|
2867
|
+
}
|
|
2868
|
+
return 0;
|
|
2869
|
+
}
|
|
2870
|
+
async function subjectExport(argv, io) {
|
|
2871
|
+
const parsed = parseArgs(argv);
|
|
2872
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "out", "json"]));
|
|
2873
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud subjects pull", 1);
|
|
2874
|
+
const target = await resolveHostedTarget(parsed);
|
|
2875
|
+
const subjectId = readRequiredSubjectId(parsed);
|
|
2876
|
+
const outputDir = requireOutDir(parsed);
|
|
2877
|
+
const response = await apiRequest(projectApiPath(target.projectId, `/subjects/${encodeURIComponent(subjectId)}/export`), {}, target.baseUrl);
|
|
2878
|
+
await writeFiles(outputDir, response.files);
|
|
2879
|
+
writeOutput({ ok: true, outputDir, files: response.files.length }, parsed, io, (result) => {
|
|
2880
|
+
const record = result;
|
|
2881
|
+
return `Exported ${record.files} file(s) to ${record.outputDir}`;
|
|
2882
|
+
});
|
|
2883
|
+
return 0;
|
|
2884
|
+
}
|
|
2885
|
+
async function subjectVisibility(argv, io, visibility) {
|
|
2886
|
+
const parsed = parseArgs(argv);
|
|
2887
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "json"]));
|
|
2888
|
+
rejectUnexpectedPositionals(parsed, `workbench cloud subjects ${visibility === "public" ? "publish" : "unpublish"}`, 1);
|
|
2889
|
+
const target = await resolveHostedTarget(parsed, { requireProjectIdentity: true });
|
|
2890
|
+
const subjectId = readRequiredSubjectId(parsed);
|
|
2891
|
+
const response = await apiRequest(projectApiPath(target.projectId, `/subjects/${encodeURIComponent(subjectId)}/publish`), { method: visibility === "public" ? "PUT" : "DELETE" }, target.baseUrl);
|
|
2892
|
+
writeOutput({ ok: true, visibility, subject: response.subject }, parsed, io, () => `${visibility === "public" ? "Published" : "Unpublished"} subject ${subjectId}.`);
|
|
2893
|
+
return 0;
|
|
2894
|
+
}
|
|
2895
|
+
async function runList(argv, io) {
|
|
2896
|
+
const parsed = parseArgs(argv);
|
|
2897
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "json"]));
|
|
2898
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud runs list", 0);
|
|
2899
|
+
const target = await resolveHostedTarget(parsed);
|
|
2900
|
+
const response = await apiRequest(projectApiPath(target.projectId, "/runs"), {}, target.baseUrl);
|
|
2901
|
+
writeOutput(response.runs, parsed, io, (runs) => runs
|
|
2902
|
+
.map((run) => `${run.id}\t${run.status}\t${run.subjectId ?? "pending"}`)
|
|
2903
|
+
.join("\n") || "No runs.");
|
|
2904
|
+
return 0;
|
|
2905
|
+
}
|
|
2906
|
+
async function runShow(argv, io) {
|
|
2907
|
+
const parsed = parseArgs(argv);
|
|
2908
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "json"]));
|
|
2909
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud runs show", 1);
|
|
2910
|
+
const target = await resolveHostedTarget(parsed, { requireProjectIdentity: true });
|
|
2911
|
+
const runId = readRequiredRunId(parsed);
|
|
2912
|
+
const response = await apiRequest(projectApiPath(target.projectId, `/runs/${encodeURIComponent(runId)}`), {}, target.baseUrl);
|
|
2913
|
+
const detail = withRunDetailUrls(target, response);
|
|
2914
|
+
writeOutput(detail, parsed, io, formatRunDetail);
|
|
2915
|
+
return 0;
|
|
2916
|
+
}
|
|
2917
|
+
async function runCancel(argv, io) {
|
|
2918
|
+
const parsed = parseArgs(argv);
|
|
2919
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "json"]));
|
|
2920
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud runs cancel", 1);
|
|
2921
|
+
const target = await resolveHostedTarget(parsed, { requireProjectIdentity: true });
|
|
2922
|
+
const runId = readRequiredRunId(parsed);
|
|
2923
|
+
const response = await apiRequest(projectApiPath(target.projectId, `/runs/${encodeURIComponent(runId)}`), { method: "DELETE" }, target.baseUrl);
|
|
2924
|
+
const run = withRunUrls(target, response.run);
|
|
2925
|
+
writeOutput(run, parsed, io, (record) => {
|
|
2926
|
+
const value = record;
|
|
2927
|
+
return [
|
|
2928
|
+
`Cancelled run ${value.id}; status ${value.status}; outcome ${value.outcome ?? "cancelled"}.`,
|
|
2929
|
+
`Open run: ${value.urls?.run ?? buildWorkbenchResourceUrls(target, { runId: value.id }).run}`,
|
|
2930
|
+
].join("\n");
|
|
2931
|
+
});
|
|
2932
|
+
return 0;
|
|
2933
|
+
}
|
|
2934
|
+
async function runWatch(argv, io) {
|
|
2935
|
+
const parsed = parseArgs(argv);
|
|
2936
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "interval-ms", "timeout-ms", "json"]));
|
|
2937
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud watch", 1);
|
|
2938
|
+
const target = await resolveHostedTarget(parsed, { requireProjectIdentity: true });
|
|
2939
|
+
const runId = readRequiredRunId(parsed);
|
|
2940
|
+
if (parsed.flags.json !== true) {
|
|
2941
|
+
io.stdout.write(`Watching run ${runId}.\n${HOSTED_WATCH_LIFECYCLE_NOTE}\n`);
|
|
2942
|
+
}
|
|
2943
|
+
const run = await watchHostedRun({
|
|
2944
|
+
parsed,
|
|
2945
|
+
target,
|
|
2946
|
+
runId,
|
|
2947
|
+
intervalMs: parsePositiveInt(parsed.flags["interval-ms"], 1000, "interval-ms"),
|
|
2948
|
+
timeoutMs: parseOptionalPositiveInt(parsed.flags["timeout-ms"], "timeout-ms"),
|
|
2949
|
+
});
|
|
2950
|
+
const outputRun = await withHostedRunFailureSummary(target, run);
|
|
2951
|
+
writeOutput(withRunUrls(target, outputRun), parsed, io, formatHostedRunResult);
|
|
2952
|
+
return hostedRunSucceeded(run) ? 0 : 1;
|
|
2953
|
+
}
|
|
2954
|
+
async function runLogs(argv, io) {
|
|
2955
|
+
const parsed = parseArgs(argv);
|
|
2956
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "json"]));
|
|
2957
|
+
rejectUnexpectedPositionals(parsed, "workbench cloud logs", 1);
|
|
2958
|
+
const target = await resolveHostedTarget(parsed);
|
|
2959
|
+
const requestedRunId = parsed.positionals[0];
|
|
2960
|
+
if (requestedRunId) {
|
|
2961
|
+
const response = await apiRequest(projectApiPath(target.projectId, `/runs/${encodeURIComponent(requestedRunId)}`), {}, target.baseUrl);
|
|
2962
|
+
writeOutput({ runId: response.run.id, jobs: response.jobs }, parsed, io, formatRunLogs);
|
|
2963
|
+
return 0;
|
|
2964
|
+
}
|
|
2965
|
+
const project = (await apiRequest(projectApiPath(target.projectId), {}, target.baseUrl)).project;
|
|
2966
|
+
const runId = project.runs.at(-1)?.id;
|
|
2967
|
+
if (!runId) {
|
|
2968
|
+
throw new UsageError("Missing RUN_ID; the benchmark has no runs.");
|
|
2969
|
+
}
|
|
2970
|
+
const jobs = project.jobs.filter((job) => job.runId === runId);
|
|
2971
|
+
writeOutput({ runId, jobs }, parsed, io, formatRunLogs);
|
|
2972
|
+
return 0;
|
|
2973
|
+
}
|
|
2974
|
+
function formatRunLogs(record) {
|
|
2975
|
+
const value = record;
|
|
2976
|
+
return (value.jobs
|
|
2977
|
+
.map((job) => `${job.id}\t${job.kind}\t${job.status}\t${job.subjectId ?? "-"}${job.error ? `\t${job.error}` : ""}`)
|
|
2978
|
+
.join("\n") || `No jobs for ${value.runId}.`);
|
|
2979
|
+
}
|
|
2980
|
+
async function openWorkbench(argv, io) {
|
|
2981
|
+
const parsed = parseArgs(argv);
|
|
2982
|
+
rejectUnknownFlags(parsed, new Set(["dir", "benchmark", "no-open", "json"]));
|
|
2983
|
+
if (parsed.positionals.length > 1) {
|
|
2984
|
+
throw new UsageError(`Unexpected argument for workbench open: ${parsed.positionals.slice(1).join(" ")}`);
|
|
2985
|
+
}
|
|
2986
|
+
const target = await resolveOpenTarget(parsed);
|
|
2987
|
+
const ref = target.openRef;
|
|
2988
|
+
const url = buildWorkbenchWebUrl(target, ref);
|
|
2989
|
+
if (parsed.flags.json === true) {
|
|
2990
|
+
writeJson({ ok: true, url }, io);
|
|
2991
|
+
}
|
|
2992
|
+
else {
|
|
2993
|
+
io.stdout.write(`${url}\n`);
|
|
2994
|
+
}
|
|
2995
|
+
if (parsed.flags["no-open"] !== true) {
|
|
2996
|
+
await openBrowser(url).catch(() => undefined);
|
|
2997
|
+
}
|
|
2998
|
+
return 0;
|
|
2999
|
+
}
|
|
3000
|
+
function buildWorkbenchWebUrl(target, ref) {
|
|
3001
|
+
const urls = buildWorkbenchResourceUrls(target);
|
|
3002
|
+
const benchmarkUrl = urls.benchmark;
|
|
3003
|
+
if (!ref) {
|
|
3004
|
+
return benchmarkUrl;
|
|
3005
|
+
}
|
|
3006
|
+
if (ref.startsWith("wb_")) {
|
|
3007
|
+
return benchmarkUrl;
|
|
3008
|
+
}
|
|
3009
|
+
if (ref.startsWith("run_")) {
|
|
3010
|
+
return buildWorkbenchResourceUrls(target, { runId: ref }).run;
|
|
3011
|
+
}
|
|
3012
|
+
return buildWorkbenchResourceUrls(target, { subjectId: ref }).subjectEvaluation;
|
|
3013
|
+
}
|
|
3014
|
+
async function resolveHostedTarget(parsed, options = {}) {
|
|
3015
|
+
if (options.sourceArg !== undefined && parsed.flags.dir !== undefined) {
|
|
3016
|
+
throw new UsageError("Use either --dir or SOURCE, not both.");
|
|
3017
|
+
}
|
|
3018
|
+
const dir = options.sourceDir
|
|
3019
|
+
? path.resolve(options.sourceDir)
|
|
3020
|
+
: resolveDir(parsed, options.sourceArg);
|
|
3021
|
+
const origin = await readWorkbenchOrigin(dir);
|
|
3022
|
+
const explicitProject = asOptionalString(parsed.flags.benchmark);
|
|
3023
|
+
const baseUrl = await effectiveBaseUrl(origin?.baseUrl);
|
|
3024
|
+
if (explicitProject && (!isRemoteProjectId(explicitProject) || options.requireProjectIdentity === true)) {
|
|
3025
|
+
const project = await resolveRemoteProject(explicitProject, baseUrl);
|
|
3026
|
+
return {
|
|
3027
|
+
projectId: project.id,
|
|
3028
|
+
owner: project.ownerUsername,
|
|
3029
|
+
projectName: project.name ?? explicitProject,
|
|
3030
|
+
dir,
|
|
3031
|
+
baseUrl,
|
|
3032
|
+
origin,
|
|
3033
|
+
};
|
|
3034
|
+
}
|
|
3035
|
+
const projectId = explicitProject ?? origin?.projectId;
|
|
3036
|
+
if (!projectId) {
|
|
3037
|
+
throw new UsageError("Missing hosted benchmark. Run workbench push, workbench clone, or pass --benchmark OWNER/BENCHMARK.");
|
|
3038
|
+
}
|
|
3039
|
+
return {
|
|
3040
|
+
projectId,
|
|
3041
|
+
...(!explicitProject && origin?.owner ? { owner: origin.owner } : {}),
|
|
3042
|
+
...(!explicitProject && origin?.project
|
|
3043
|
+
? { projectName: origin.project }
|
|
3044
|
+
: {}),
|
|
3045
|
+
dir,
|
|
3046
|
+
baseUrl,
|
|
3047
|
+
origin,
|
|
3048
|
+
};
|
|
3049
|
+
}
|
|
3050
|
+
async function resolveOpenTarget(parsed) {
|
|
3051
|
+
const ref = parsed.positionals[0];
|
|
3052
|
+
if (ref &&
|
|
3053
|
+
!ref.startsWith("run_") &&
|
|
3054
|
+
!ref.startsWith("subject_")) {
|
|
3055
|
+
const baseUrl = await effectiveBaseUrl();
|
|
3056
|
+
if (ref.includes("/")) {
|
|
3057
|
+
const parsedRef = parseBenchmarkRef(ref);
|
|
3058
|
+
const project = await apiRequest(publicProjectApiPath(parsedRef), {}, baseUrl);
|
|
3059
|
+
return {
|
|
3060
|
+
projectId: project.benchmark.id,
|
|
3061
|
+
owner: project.benchmark.ownerUsername ?? parsedRef.owner,
|
|
3062
|
+
projectName: project.benchmark.name ?? parsedRef.project,
|
|
3063
|
+
dir: resolveDir(parsed),
|
|
3064
|
+
baseUrl,
|
|
3065
|
+
};
|
|
3066
|
+
}
|
|
3067
|
+
const project = await resolveRemoteProject(ref, baseUrl);
|
|
3068
|
+
return {
|
|
3069
|
+
projectId: project.id,
|
|
3070
|
+
owner: project.ownerUsername,
|
|
3071
|
+
projectName: project.name ?? ref,
|
|
3072
|
+
dir: resolveDir(parsed),
|
|
3073
|
+
baseUrl,
|
|
3074
|
+
};
|
|
3075
|
+
}
|
|
3076
|
+
return {
|
|
3077
|
+
...(await resolveHostedTarget(parsed, { requireProjectIdentity: true })),
|
|
3078
|
+
...(ref ? { openRef: ref } : {}),
|
|
3079
|
+
};
|
|
3080
|
+
}
|
|
3081
|
+
function buildWorkbenchResourceUrls(target, refs = {}) {
|
|
3082
|
+
if (!target.owner || !target.projectName) {
|
|
3083
|
+
throw new UsageError(`Cannot build Workbench Cloud URL for ${target.projectId} without owner username and benchmark name.`);
|
|
3084
|
+
}
|
|
3085
|
+
const projectRef = `${encodeURIComponent(target.owner)}/${encodeURIComponent(target.projectName)}`;
|
|
3086
|
+
const benchmark = `${target.baseUrl}/benchmarks/${projectRef}`;
|
|
3087
|
+
const urls = { benchmark };
|
|
3088
|
+
if (refs.runId) {
|
|
3089
|
+
urls.run = `${benchmark}/runs/${encodeURIComponent(refs.runId)}`;
|
|
3090
|
+
urls.traces = urls.run;
|
|
3091
|
+
}
|
|
3092
|
+
if (refs.subjectId) {
|
|
3093
|
+
urls.subjectEvaluation = `${benchmark}/subject/${encodeURIComponent(refs.subjectId)}/evaluation`;
|
|
3094
|
+
}
|
|
3095
|
+
return urls;
|
|
3096
|
+
}
|
|
3097
|
+
function projectApiPath(projectRef, suffix = "") {
|
|
3098
|
+
return `/api/workbench/benchmarks/${encodeURIComponent(projectRef)}${suffix}`;
|
|
3099
|
+
}
|
|
3100
|
+
function benchmarkApiPath(benchmarkRef) {
|
|
3101
|
+
if (benchmarkRef.includes("/")) {
|
|
3102
|
+
return publicProjectApiPath(parseBenchmarkRef(benchmarkRef));
|
|
3103
|
+
}
|
|
3104
|
+
return projectApiPath(benchmarkRef);
|
|
3105
|
+
}
|
|
3106
|
+
function publicProjectApiPath(ref) {
|
|
3107
|
+
return `/api/workbench/public/benchmarks/${encodeURIComponent(ref.owner)}/${encodeURIComponent(ref.project)}`;
|
|
3108
|
+
}
|
|
3109
|
+
function publicProjectSourceApiPath(ref) {
|
|
3110
|
+
return `${publicProjectApiPath(ref)}/source`;
|
|
3111
|
+
}
|
|
3112
|
+
function readRequiredBenchmarkRef(parsed) {
|
|
3113
|
+
const ref = parsed.positionals[0];
|
|
3114
|
+
if (!ref) {
|
|
3115
|
+
throw new UsageError("Missing required OWNER/BENCHMARK ref.");
|
|
3116
|
+
}
|
|
3117
|
+
return parseBenchmarkRef(ref);
|
|
3118
|
+
}
|
|
3119
|
+
function parseBenchmarkRef(value) {
|
|
3120
|
+
const [namePart, versionRef, extraRef] = value.split("@");
|
|
3121
|
+
if (extraRef !== undefined || !namePart) {
|
|
3122
|
+
throw new UsageError("Benchmark refs must use OWNER/BENCHMARK[@REF].");
|
|
3123
|
+
}
|
|
3124
|
+
const [owner, project, extra] = namePart.split("/");
|
|
3125
|
+
if (!owner || !project || extra !== undefined) {
|
|
3126
|
+
throw new UsageError("Benchmark refs must use OWNER/BENCHMARK[@REF].");
|
|
3127
|
+
}
|
|
3128
|
+
return { owner, project, ...(versionRef ? { ref: versionRef } : {}) };
|
|
3129
|
+
}
|
|
3130
|
+
function formatBenchmarkRef(ref) {
|
|
3131
|
+
return `${ref.owner}/${ref.project}${ref.ref ? `@${ref.ref}` : ""}`;
|
|
3132
|
+
}
|
|
3133
|
+
async function resolveRemoteProject(projectRef, baseUrl) {
|
|
3134
|
+
if (projectRef.includes("/")) {
|
|
3135
|
+
const ref = parseBenchmarkRef(projectRef);
|
|
3136
|
+
const response = await apiRequest(publicProjectApiPath(ref), {}, baseUrl);
|
|
3137
|
+
return response.benchmark;
|
|
3138
|
+
}
|
|
3139
|
+
const response = await apiRequest(projectApiPath(projectRef), {}, baseUrl);
|
|
3140
|
+
return response.benchmark;
|
|
3141
|
+
}
|
|
3142
|
+
function formatProjectRef(project) {
|
|
3143
|
+
return project.name ? `${project.name} (${project.id})` : project.id;
|
|
3144
|
+
}
|
|
3145
|
+
function withRunUrls(target, run) {
|
|
3146
|
+
return {
|
|
3147
|
+
...run,
|
|
3148
|
+
urls: buildWorkbenchResourceUrls(target, {
|
|
3149
|
+
runId: run.id,
|
|
3150
|
+
subjectId: run.subjectId,
|
|
3151
|
+
}),
|
|
3152
|
+
};
|
|
3153
|
+
}
|
|
3154
|
+
function withRunDetailUrls(target, detail) {
|
|
3155
|
+
const subjectId = detail.run.subjectId ?? detail.jobs.find((job) => job.subjectId)?.subjectId ?? null;
|
|
3156
|
+
const run = withRunUrls(target, { ...detail.run, subjectId });
|
|
3157
|
+
return {
|
|
3158
|
+
run,
|
|
3159
|
+
jobs: detail.jobs,
|
|
3160
|
+
urls: run.urls ?? buildWorkbenchResourceUrls(target, { runId: run.id }),
|
|
3161
|
+
};
|
|
3162
|
+
}
|
|
3163
|
+
function sourceFileCount(source) {
|
|
3164
|
+
return source.sourceFiles.length;
|
|
3165
|
+
}
|
|
3166
|
+
function hostedProjectSourceRequest(source) {
|
|
3167
|
+
const { network, resources } = hostedEnvironmentOptions(source);
|
|
3168
|
+
return {
|
|
3169
|
+
source: source.specSource,
|
|
3170
|
+
subjectFiles: source.subjectFiles,
|
|
3171
|
+
engineResolveFiles: hostedEngineResolveFiles(source),
|
|
3172
|
+
engineResolveBinding: engineResolveBindingForSpec(source.spec),
|
|
3173
|
+
adapterFiles: source.adapterFiles,
|
|
3174
|
+
dockerfile: source.dockerfile,
|
|
3175
|
+
runtimeDockerfile: source.runtimeDockerfile,
|
|
3176
|
+
runtimeFiles: source.dockerfileFiles,
|
|
3177
|
+
network,
|
|
3178
|
+
resources,
|
|
3179
|
+
};
|
|
3180
|
+
}
|
|
3181
|
+
function hostedEngineResolveFiles(source) {
|
|
3182
|
+
return [
|
|
3183
|
+
...source.engineResolveFiles,
|
|
3184
|
+
{
|
|
3185
|
+
path: WORKBENCH_ADAPTER_RESULT_FILE,
|
|
3186
|
+
content: `${JSON.stringify({
|
|
3187
|
+
protocol: WORKBENCH_ADAPTER_RESULT_PROTOCOL,
|
|
3188
|
+
operation: "engine.resolve",
|
|
3189
|
+
ok: true,
|
|
3190
|
+
value: {
|
|
3191
|
+
cases: source.engineCases,
|
|
3192
|
+
...(source.engineResolveEnvironment
|
|
3193
|
+
? { environment: source.engineResolveEnvironment }
|
|
3194
|
+
: {}),
|
|
3195
|
+
},
|
|
3196
|
+
feedback: {
|
|
3197
|
+
path: source.engineResolveFingerprintPath,
|
|
3198
|
+
},
|
|
3199
|
+
}, null, 2)}\n`,
|
|
3200
|
+
},
|
|
3201
|
+
];
|
|
3202
|
+
}
|
|
3203
|
+
function isRemoteProjectId(value) {
|
|
3204
|
+
return /^wb_[a-f0-9]{12}$/u.test(value);
|
|
3205
|
+
}
|
|
3206
|
+
function hostedEnvironmentOptions(source) {
|
|
3207
|
+
const rawResources = source.spec.environment.resources ?? {};
|
|
3208
|
+
return {
|
|
3209
|
+
network: source.spec.environment.network?.egress === "open"
|
|
3210
|
+
? "on"
|
|
3211
|
+
: "off",
|
|
3212
|
+
resources: {
|
|
3213
|
+
cpu: typeof rawResources.cpu === "number" ? rawResources.cpu : undefined,
|
|
3214
|
+
memoryGb: typeof rawResources.memoryGb === "number"
|
|
3215
|
+
? rawResources.memoryGb
|
|
3216
|
+
: undefined,
|
|
3217
|
+
diskGb: typeof rawResources.diskGb === "number" ? rawResources.diskGb : undefined,
|
|
3218
|
+
timeoutMinutes: typeof rawResources.timeoutMinutes === "number"
|
|
3219
|
+
? rawResources.timeoutMinutes
|
|
3220
|
+
: undefined,
|
|
3221
|
+
},
|
|
3222
|
+
};
|
|
3223
|
+
}
|
|
3224
|
+
async function watchHostedRun(args) {
|
|
3225
|
+
const deadline = args.timeoutMs === undefined ? undefined : Date.now() + args.timeoutMs;
|
|
3226
|
+
let lastRun = null;
|
|
3227
|
+
while (true) {
|
|
3228
|
+
const response = await apiRequest(projectApiPath(args.target.projectId, `/runs/${encodeURIComponent(args.runId)}`), {}, args.target.baseUrl);
|
|
3229
|
+
lastRun = response.run;
|
|
3230
|
+
if (response.run.status === "finished") {
|
|
3231
|
+
return response.run;
|
|
3232
|
+
}
|
|
3233
|
+
if (deadline !== undefined && Date.now() > deadline) {
|
|
3234
|
+
throw new Error(`Timed out waiting for run ${args.runId}; last status was ${lastRun?.status ?? "unknown"}.`);
|
|
3235
|
+
}
|
|
3236
|
+
await sleep(args.intervalMs);
|
|
3237
|
+
}
|
|
3238
|
+
}
|
|
3239
|
+
function formatHostedRunResult(run) {
|
|
3240
|
+
const summary = `Run ${run.id} reached ${run.status}; ${run.outcome ? `outcome ${run.outcome}; ` : ""}subject ${run.subjectId ?? "pending"}; ${run.completedJobCount ?? 0}/${run.jobCount ?? 0} jobs completed.`;
|
|
3241
|
+
return [
|
|
3242
|
+
run.error ? `${summary}\nError: ${run.error}` : summary,
|
|
3243
|
+
...(run.urls?.run ? [`Open run: ${run.urls.run}`] : []),
|
|
3244
|
+
...(run.urls?.subjectEvaluation
|
|
3245
|
+
? [`Open evaluation: ${run.urls.subjectEvaluation}`]
|
|
3246
|
+
: []),
|
|
3247
|
+
].join("\n");
|
|
3248
|
+
}
|
|
3249
|
+
function formatHostedRunStarted(run, fallbackWorkflow) {
|
|
3250
|
+
return [
|
|
3251
|
+
`Started ${run.workflow ?? fallbackWorkflow} run ${run.id}; ${run.subjectId ? `subject ${run.subjectId}` : `${run.jobCount ?? 0} jobs queued`}.`,
|
|
3252
|
+
...(run.urls?.run ? [`Open run: ${run.urls.run}`] : []),
|
|
3253
|
+
"",
|
|
3254
|
+
].join("\n");
|
|
3255
|
+
}
|
|
3256
|
+
function formatRunDetail(record) {
|
|
3257
|
+
const detail = record;
|
|
3258
|
+
const { run, jobs, urls } = detail;
|
|
3259
|
+
const cost = sumJobCostUsd(jobs);
|
|
3260
|
+
const firstFailedJob = jobs.find((job) => job.status === "failed" && job.error);
|
|
3261
|
+
const subjectId = run.subjectId ?? jobs.find((job) => job.subjectId)?.subjectId ?? null;
|
|
3262
|
+
return [
|
|
3263
|
+
`Run ${run.id}: ${run.status}${run.outcome ? ` (${run.outcome})` : ""}`,
|
|
3264
|
+
`Workflow: ${run.workflow ?? "improve"}`,
|
|
3265
|
+
`Subject: ${subjectId ?? "pending"}`,
|
|
3266
|
+
`Samples: ${run.samples ?? 0}`,
|
|
3267
|
+
`Attempts: ${run.attemptsExecuted ?? 0}/${run.attemptsRequested ?? run.attemptsExecuted ?? 0}`,
|
|
3268
|
+
`Jobs: ${run.completedJobCount ?? jobs.filter(isTerminalRunJob).length}/${run.jobCount ?? jobs.length} completed${run.failedJobCount ? `; ${run.failedJobCount} failed` : ""}`,
|
|
3269
|
+
...(typeof run.durationMs === "number"
|
|
3270
|
+
? [`Duration: ${formatDurationMs(run.durationMs)}`]
|
|
3271
|
+
: []),
|
|
3272
|
+
...(cost > 0 ? [`Cost: ${formatUsd(cost)}`] : []),
|
|
3273
|
+
...(firstFailedJob?.error
|
|
3274
|
+
? [`First failed job ${firstFailedJob.id}: ${firstFailedJob.error}`]
|
|
3275
|
+
: []),
|
|
3276
|
+
`Open run: ${urls.run ?? urls.benchmark}`,
|
|
3277
|
+
...(urls.subjectEvaluation ? [`Open evaluation: ${urls.subjectEvaluation}`] : []),
|
|
3278
|
+
...(jobs.length > 0 ? ["", "Jobs:", ...jobs.map(formatRunJobLine)] : []),
|
|
3279
|
+
].join("\n");
|
|
3280
|
+
}
|
|
3281
|
+
function formatRunJobLine(job) {
|
|
3282
|
+
return [
|
|
3283
|
+
job.id,
|
|
3284
|
+
readRunJobPurpose(job) ?? job.kind ?? "job",
|
|
3285
|
+
job.status,
|
|
3286
|
+
job.subjectId ?? "-",
|
|
3287
|
+
job.error ?? "",
|
|
3288
|
+
].filter((value, index) => index < 4 || value !== "").join("\t");
|
|
3289
|
+
}
|
|
3290
|
+
function isTerminalRunJob(job) {
|
|
3291
|
+
return job.status === "succeeded" || job.status === "failed" || job.status === "cancelled";
|
|
3292
|
+
}
|
|
3293
|
+
function readRunJobPurpose(job) {
|
|
3294
|
+
const input = readRecord(job.input);
|
|
3295
|
+
const execution = readRecord(input?.execution);
|
|
3296
|
+
const purpose = execution?.purpose;
|
|
3297
|
+
return typeof purpose === "string" && purpose ? purpose : null;
|
|
3298
|
+
}
|
|
3299
|
+
function sumJobCostUsd(jobs) {
|
|
3300
|
+
const sum = jobs.reduce((total, job) => total + costUsdFromUsage(readRecord(job.output)?.usage), 0);
|
|
3301
|
+
return Number.isFinite(sum) ? Math.round(sum * 1_000_000) / 1_000_000 : 0;
|
|
3302
|
+
}
|
|
3303
|
+
function costUsdFromUsage(value) {
|
|
3304
|
+
const usage = readRecord(value);
|
|
3305
|
+
if (!usage) {
|
|
3306
|
+
return 0;
|
|
3307
|
+
}
|
|
3308
|
+
const direct = readFiniteNumber(usage.costUsd);
|
|
3309
|
+
if (direct !== null) {
|
|
3310
|
+
return direct;
|
|
3311
|
+
}
|
|
3312
|
+
return ["total", "optimizer", "runner", "engine"].reduce((sum, key) => {
|
|
3313
|
+
const nested = readRecord(usage[key]);
|
|
3314
|
+
return sum + (readFiniteNumber(nested?.costUsd) ?? 0);
|
|
3315
|
+
}, 0);
|
|
3316
|
+
}
|
|
3317
|
+
function readRecord(value) {
|
|
3318
|
+
return value && typeof value === "object" && !Array.isArray(value)
|
|
3319
|
+
? value
|
|
3320
|
+
: null;
|
|
3321
|
+
}
|
|
3322
|
+
function readFiniteNumber(value) {
|
|
3323
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
3324
|
+
}
|
|
3325
|
+
function formatDurationMs(durationMs) {
|
|
3326
|
+
if (durationMs < 1000) {
|
|
3327
|
+
return `${Math.max(0, Math.round(durationMs))}ms`;
|
|
3328
|
+
}
|
|
3329
|
+
const seconds = durationMs / 1000;
|
|
3330
|
+
if (seconds < 60) {
|
|
3331
|
+
return `${seconds.toFixed(seconds < 10 ? 1 : 0)}s`;
|
|
3332
|
+
}
|
|
3333
|
+
const minutes = Math.floor(seconds / 60);
|
|
3334
|
+
const remainingSeconds = Math.round(seconds % 60);
|
|
3335
|
+
return `${minutes}m ${remainingSeconds}s`;
|
|
3336
|
+
}
|
|
3337
|
+
function formatUsd(value) {
|
|
3338
|
+
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
3339
|
+
}
|
|
3340
|
+
function shortDigest(value) {
|
|
3341
|
+
return value.length > 12 ? value.slice(0, 12) : value;
|
|
3342
|
+
}
|
|
3343
|
+
async function withHostedRunFailureSummary(target, run) {
|
|
3344
|
+
if (hostedRunSucceeded(run) || run.error || (run.failedJobCount ?? 0) <= 0) {
|
|
3345
|
+
return run;
|
|
3346
|
+
}
|
|
3347
|
+
const error = await readHostedRunFailureSummary(target, run.id);
|
|
3348
|
+
return error ? { ...run, error } : run;
|
|
3349
|
+
}
|
|
3350
|
+
async function readHostedRunFailureSummary(target, runId) {
|
|
3351
|
+
try {
|
|
3352
|
+
const project = await apiRequest(projectApiPath(target.projectId), {}, target.baseUrl);
|
|
3353
|
+
const failed = project.benchmark.jobs.find((job) => job.runId === runId && job.status === "failed" && job.error);
|
|
3354
|
+
return failed?.error ? `First failed job ${failed.id}: ${failed.error}` : null;
|
|
3355
|
+
}
|
|
3356
|
+
catch {
|
|
3357
|
+
return null;
|
|
3358
|
+
}
|
|
3359
|
+
}
|
|
3360
|
+
function hostedRunSucceeded(run) {
|
|
3361
|
+
if (run.status !== "finished") {
|
|
3362
|
+
return false;
|
|
3363
|
+
}
|
|
3364
|
+
if ((run.failedJobCount ?? 0) > 0) {
|
|
3365
|
+
return false;
|
|
3366
|
+
}
|
|
3367
|
+
return run.outcome == null || run.outcome === "ok";
|
|
3368
|
+
}
|
|
3369
|
+
async function readWorkbenchOrigin(dir) {
|
|
3370
|
+
try {
|
|
3371
|
+
const parsed = JSON.parse(await fs.readFile(workbenchOriginPath(dir), "utf8"));
|
|
3372
|
+
if (!parsed.projectId ||
|
|
3373
|
+
!parsed.baseUrl ||
|
|
3374
|
+
!parsed.owner ||
|
|
3375
|
+
!parsed.project ||
|
|
3376
|
+
typeof parsed.writable !== "boolean") {
|
|
3377
|
+
throw new UsageError(`Workbench origin is malformed: ${workbenchOriginPath(dir)}`);
|
|
3378
|
+
}
|
|
3379
|
+
return {
|
|
3380
|
+
baseUrl: normalizeBaseUrl(parsed.baseUrl),
|
|
3381
|
+
owner: parsed.owner,
|
|
3382
|
+
project: parsed.project,
|
|
3383
|
+
projectId: parsed.projectId,
|
|
3384
|
+
writable: parsed.writable,
|
|
3385
|
+
...(parsed.sourceRevisionId ? { sourceRevisionId: parsed.sourceRevisionId } : {}),
|
|
3386
|
+
...(parsed.sourceFingerprint ? { sourceFingerprint: parsed.sourceFingerprint } : {}),
|
|
3387
|
+
...(parsed.upstream ? { upstream: parsed.upstream } : {}),
|
|
3388
|
+
linkedAt: parsed.linkedAt ?? new Date(0).toISOString(),
|
|
3389
|
+
};
|
|
3390
|
+
}
|
|
3391
|
+
catch (error) {
|
|
3392
|
+
if (error.code === "ENOENT") {
|
|
3393
|
+
return null;
|
|
3394
|
+
}
|
|
3395
|
+
throw error;
|
|
3396
|
+
}
|
|
3397
|
+
}
|
|
3398
|
+
async function requireWorkbenchOrigin(dir) {
|
|
3399
|
+
const origin = await readWorkbenchOrigin(dir);
|
|
3400
|
+
if (!origin) {
|
|
3401
|
+
throw new UsageError("Missing Workbench origin. Run workbench push or workbench clone first.");
|
|
3402
|
+
}
|
|
3403
|
+
return origin;
|
|
3404
|
+
}
|
|
3405
|
+
async function writeWorkbenchOrigin(dir, input) {
|
|
3406
|
+
const origin = {
|
|
3407
|
+
...input,
|
|
3408
|
+
baseUrl: normalizeBaseUrl(input.baseUrl),
|
|
3409
|
+
linkedAt: input.linkedAt ?? new Date().toISOString(),
|
|
3410
|
+
};
|
|
3411
|
+
const filePath = workbenchOriginPath(dir);
|
|
3412
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
3413
|
+
await fs.writeFile(filePath, `${JSON.stringify(origin, null, 2)}\n`);
|
|
3414
|
+
return origin;
|
|
3415
|
+
}
|
|
3416
|
+
function workbenchOriginPath(dir) {
|
|
3417
|
+
return path.join(dir, ".workbench", "origin.json");
|
|
3418
|
+
}
|
|
3419
|
+
async function effectiveBaseUrl(preferred) {
|
|
3420
|
+
const config = await loadConfig();
|
|
3421
|
+
return normalizeBaseUrl(process.env.WORKBENCH_API_URL ??
|
|
3422
|
+
preferred ??
|
|
3423
|
+
config.baseUrl ??
|
|
3424
|
+
DEFAULT_BASE_URL);
|
|
3425
|
+
}
|
|
3426
|
+
function readOptionalSubjectId(parsed) {
|
|
3427
|
+
return asOptionalString(parsed.flags.subject) ?? parsed.positionals[0];
|
|
3428
|
+
}
|
|
3429
|
+
function readRequiredSubjectId(parsed) {
|
|
3430
|
+
const subjectId = readOptionalSubjectId(parsed);
|
|
3431
|
+
if (!subjectId) {
|
|
3432
|
+
throw new UsageError("Missing required SUBJECT_ID.");
|
|
3433
|
+
}
|
|
3434
|
+
return subjectId;
|
|
3435
|
+
}
|
|
3436
|
+
function readRequiredRunId(parsed) {
|
|
3437
|
+
const runId = parsed.positionals[0];
|
|
3438
|
+
if (!runId) {
|
|
3439
|
+
throw new UsageError("Missing required RUN_ID.");
|
|
3440
|
+
}
|
|
3441
|
+
return runId;
|
|
3442
|
+
}
|
|
3443
|
+
function requireOutDir(parsed) {
|
|
3444
|
+
const output = asOptionalString(parsed.flags.out);
|
|
3445
|
+
if (!output) {
|
|
3446
|
+
throw new UsageError("Missing required --out.");
|
|
3447
|
+
}
|
|
3448
|
+
return output;
|
|
3449
|
+
}
|
|
3450
|
+
async function apiRequest(apiPath, options = {}, baseUrlOverride) {
|
|
3451
|
+
const config = await loadConfig();
|
|
3452
|
+
const baseUrl = normalizeBaseUrl(baseUrlOverride ??
|
|
3453
|
+
process.env.WORKBENCH_API_URL ??
|
|
3454
|
+
config.baseUrl ??
|
|
3455
|
+
DEFAULT_BASE_URL);
|
|
3456
|
+
const response = await fetch(`${baseUrl}${apiPath}`, {
|
|
3457
|
+
method: options.method ?? "GET",
|
|
3458
|
+
headers: {
|
|
3459
|
+
"content-type": "application/json",
|
|
3460
|
+
...(config.accessToken
|
|
3461
|
+
? { authorization: `Bearer ${config.accessToken}` }
|
|
3462
|
+
: {}),
|
|
3463
|
+
},
|
|
3464
|
+
body: options.body == null ? undefined : JSON.stringify(options.body),
|
|
3465
|
+
});
|
|
3466
|
+
if (!response.ok) {
|
|
3467
|
+
const text = await response.text();
|
|
3468
|
+
throw new Error(readResponseError(text) ||
|
|
3469
|
+
`Request failed with status ${response.status}.`);
|
|
3470
|
+
}
|
|
3471
|
+
return (await response.json());
|
|
3472
|
+
}
|
|
3473
|
+
async function uploadAdapterConnection(bundle) {
|
|
3474
|
+
const config = await loadConfig();
|
|
3475
|
+
if (!config.accessToken) {
|
|
3476
|
+
return { status: "skipped", reason: "not_authenticated" };
|
|
3477
|
+
}
|
|
3478
|
+
await apiRequest(adapterConnectionApiPath(bundle), {
|
|
3479
|
+
method: "PUT",
|
|
3480
|
+
body: { bundle },
|
|
3481
|
+
});
|
|
3482
|
+
return { status: "connected" };
|
|
3483
|
+
}
|
|
3484
|
+
async function deleteAdapterConnection(target) {
|
|
3485
|
+
const config = await loadConfig();
|
|
3486
|
+
if (!config.accessToken) {
|
|
3487
|
+
return { status: "skipped", reason: "not_authenticated" };
|
|
3488
|
+
}
|
|
3489
|
+
await apiRequest(adapterConnectionApiPath(target), { method: "DELETE" });
|
|
3490
|
+
return { status: "disconnected" };
|
|
3491
|
+
}
|
|
3492
|
+
function adapterConnectionApiPath(target) {
|
|
3493
|
+
const query = new URLSearchParams({
|
|
3494
|
+
profile: target.profile,
|
|
3495
|
+
...(target.slot ? { slot: target.slot } : {}),
|
|
3496
|
+
});
|
|
3497
|
+
return `/api/workbench/auth/adapters/${encodeURIComponent(target.adapterId)}?${query}`;
|
|
3498
|
+
}
|
|
3499
|
+
async function requestDeviceAuthorization(baseUrl) {
|
|
3500
|
+
const response = await fetch(`${normalizeBaseUrl(baseUrl)}/api/oauth/device/code`, {
|
|
3501
|
+
method: "POST",
|
|
3502
|
+
});
|
|
3503
|
+
if (!response.ok) {
|
|
3504
|
+
throw new Error(readResponseError(await response.text()) ||
|
|
3505
|
+
"Unable to start Workbench login.");
|
|
3506
|
+
}
|
|
3507
|
+
return (await response.json());
|
|
3508
|
+
}
|
|
3509
|
+
async function pollDeviceToken(baseUrl, authorization) {
|
|
3510
|
+
const deadline = Date.now() + authorization.expires_in * 1000;
|
|
3511
|
+
const intervalMs = Math.max(0, authorization.interval) * 1000;
|
|
3512
|
+
while (Date.now() < deadline) {
|
|
3513
|
+
await sleep(intervalMs);
|
|
3514
|
+
const response = await fetch(`${normalizeBaseUrl(baseUrl)}/api/oauth/token`, {
|
|
3515
|
+
method: "POST",
|
|
3516
|
+
headers: {
|
|
3517
|
+
"content-type": "application/json",
|
|
3518
|
+
},
|
|
3519
|
+
body: JSON.stringify({
|
|
3520
|
+
grant_type: "urn:ietf:params:oauth:grant-type:device_code",
|
|
3521
|
+
device_code: authorization.device_code,
|
|
3522
|
+
}),
|
|
3523
|
+
});
|
|
3524
|
+
const body = await response.text();
|
|
3525
|
+
if (response.ok) {
|
|
3526
|
+
return JSON.parse(body);
|
|
3527
|
+
}
|
|
3528
|
+
const error = readOAuthError(body);
|
|
3529
|
+
if (error === "authorization_pending") {
|
|
3530
|
+
continue;
|
|
3531
|
+
}
|
|
3532
|
+
throw new Error(error || "Workbench login failed.");
|
|
3533
|
+
}
|
|
3534
|
+
throw new Error("Workbench login expired before approval.");
|
|
3535
|
+
}
|
|
3536
|
+
async function openBrowser(url) {
|
|
3537
|
+
const command = process.platform === "darwin"
|
|
3538
|
+
? "open"
|
|
3539
|
+
: process.platform === "win32"
|
|
3540
|
+
? "cmd"
|
|
3541
|
+
: "xdg-open";
|
|
3542
|
+
const args = process.platform === "win32" ? ["/c", "start", "", url] : [url];
|
|
3543
|
+
await new Promise((resolve, reject) => {
|
|
3544
|
+
const child = spawn(command, args, {
|
|
3545
|
+
detached: true,
|
|
3546
|
+
stdio: "ignore",
|
|
3547
|
+
});
|
|
3548
|
+
child.on("error", reject);
|
|
3549
|
+
child.on("spawn", () => {
|
|
3550
|
+
child.unref();
|
|
3551
|
+
resolve();
|
|
3552
|
+
});
|
|
3553
|
+
});
|
|
3554
|
+
}
|
|
3555
|
+
async function sleep(milliseconds) {
|
|
3556
|
+
await new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
3557
|
+
}
|
|
3558
|
+
function readResponseError(text) {
|
|
3559
|
+
try {
|
|
3560
|
+
const body = JSON.parse(text);
|
|
3561
|
+
return typeof body.message === "string"
|
|
3562
|
+
? body.message
|
|
3563
|
+
: typeof body.error === "string"
|
|
3564
|
+
? body.error
|
|
3565
|
+
: "";
|
|
3566
|
+
}
|
|
3567
|
+
catch {
|
|
3568
|
+
return text;
|
|
3569
|
+
}
|
|
3570
|
+
}
|
|
3571
|
+
function readOAuthError(text) {
|
|
3572
|
+
try {
|
|
3573
|
+
const body = JSON.parse(text);
|
|
3574
|
+
return typeof body.error === "string" ? body.error : "";
|
|
3575
|
+
}
|
|
3576
|
+
catch {
|
|
3577
|
+
return "";
|
|
3578
|
+
}
|
|
3579
|
+
}
|
|
3580
|
+
function parseArgs(argv) {
|
|
3581
|
+
const positionals = [];
|
|
3582
|
+
const flags = {};
|
|
3583
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
3584
|
+
const arg = argv[index];
|
|
3585
|
+
if (!arg) {
|
|
3586
|
+
continue;
|
|
3587
|
+
}
|
|
3588
|
+
if (!arg.startsWith("--")) {
|
|
3589
|
+
positionals.push(arg);
|
|
3590
|
+
continue;
|
|
3591
|
+
}
|
|
3592
|
+
const equalsIndex = arg.indexOf("=");
|
|
3593
|
+
if (equalsIndex > 0) {
|
|
3594
|
+
flags[arg.slice(2, equalsIndex)] = arg.slice(equalsIndex + 1);
|
|
3595
|
+
continue;
|
|
3596
|
+
}
|
|
3597
|
+
const key = arg.slice(2);
|
|
3598
|
+
const next = argv[index + 1];
|
|
3599
|
+
if (next && !next.startsWith("--")) {
|
|
3600
|
+
flags[key] = next;
|
|
3601
|
+
index += 1;
|
|
3602
|
+
}
|
|
3603
|
+
else {
|
|
3604
|
+
flags[key] = true;
|
|
3605
|
+
}
|
|
3606
|
+
}
|
|
3607
|
+
return { positionals, flags };
|
|
3608
|
+
}
|
|
3609
|
+
function requireFlag(parsed, key) {
|
|
3610
|
+
const value = parsed.flags[key];
|
|
3611
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
3612
|
+
throw new UsageError(`Missing required --${key}.`);
|
|
3613
|
+
}
|
|
3614
|
+
return value;
|
|
3615
|
+
}
|
|
3616
|
+
function rejectUnknownFlags(parsed, allowed) {
|
|
3617
|
+
const unknown = Object.keys(parsed.flags).filter((flag) => !allowed.has(flag));
|
|
3618
|
+
if (unknown.length > 0) {
|
|
3619
|
+
throw new UsageError(`Unsupported flag${unknown.length === 1 ? "" : "s"}: ${unknown.map((flag) => `--${flag}`).join(", ")}.`);
|
|
3620
|
+
}
|
|
3621
|
+
}
|
|
3622
|
+
function rejectUnexpectedPositionals(parsed, command, max) {
|
|
3623
|
+
if (parsed.positionals.length <= max) {
|
|
3624
|
+
return;
|
|
3625
|
+
}
|
|
3626
|
+
throw new UsageError(`Unexpected argument for ${command}: ${parsed.positionals.slice(max).join(" ")}`);
|
|
3627
|
+
}
|
|
3628
|
+
function readInitSelection(parsed) {
|
|
3629
|
+
const selections = ["skill", "command"].flatMap((kind) => parsed.flags[kind] === undefined
|
|
3630
|
+
? []
|
|
3631
|
+
: [{ kind, value: parsed.flags[kind] }]);
|
|
3632
|
+
if (selections.length !== 1) {
|
|
3633
|
+
throw new UsageError("Specify exactly one of --skill NAME or --command NAME.");
|
|
3634
|
+
}
|
|
3635
|
+
const { kind, value } = selections[0];
|
|
3636
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
3637
|
+
throw new UsageError(`Missing NAME for --${kind}.`);
|
|
3638
|
+
}
|
|
3639
|
+
return { kind, name: value };
|
|
3640
|
+
}
|
|
3641
|
+
function readInitAgent(parsed, kind) {
|
|
3642
|
+
const agent = asOptionalString(parsed.flags.agent);
|
|
3643
|
+
if (kind === "command") {
|
|
3644
|
+
if (agent) {
|
|
3645
|
+
throw new UsageError("--agent applies only to --skill.");
|
|
3646
|
+
}
|
|
3647
|
+
return undefined;
|
|
3648
|
+
}
|
|
3649
|
+
if (agent && /^[a-z][a-z0-9-]*$/u.test(agent)) {
|
|
3650
|
+
return agent;
|
|
3651
|
+
}
|
|
3652
|
+
throw new UsageError(`--agent is required for --${kind} and must be a lowercase adapter id.`);
|
|
3653
|
+
}
|
|
3654
|
+
function asOptionalString(value) {
|
|
3655
|
+
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
3656
|
+
}
|
|
3657
|
+
function readOptionalStringFlag(value, name) {
|
|
3658
|
+
if (value == null || value === false) {
|
|
3659
|
+
return undefined;
|
|
3660
|
+
}
|
|
3661
|
+
if (typeof value !== "string" || value.trim() === "") {
|
|
3662
|
+
throw new UsageError(`--${name} requires a value.`);
|
|
3663
|
+
}
|
|
3664
|
+
return value;
|
|
3665
|
+
}
|
|
3666
|
+
function parsePositiveInt(value, fallback, name) {
|
|
3667
|
+
if (value == null || value === false) {
|
|
3668
|
+
return fallback;
|
|
3669
|
+
}
|
|
3670
|
+
const raw = String(value);
|
|
3671
|
+
if (!/^[1-9]\d*$/u.test(raw)) {
|
|
3672
|
+
throw new UsageError(`--${name} must be a positive integer.`);
|
|
3673
|
+
}
|
|
3674
|
+
const parsed = Number(raw);
|
|
3675
|
+
if (!Number.isSafeInteger(parsed)) {
|
|
3676
|
+
throw new UsageError(`--${name} must be a positive integer.`);
|
|
3677
|
+
}
|
|
3678
|
+
return parsed;
|
|
3679
|
+
}
|
|
3680
|
+
function parseOptionalPositiveInt(value, name) {
|
|
3681
|
+
if (value == null || value === false) {
|
|
3682
|
+
return undefined;
|
|
3683
|
+
}
|
|
3684
|
+
return parsePositiveInt(value, 1, name);
|
|
3685
|
+
}
|
|
3686
|
+
function parsePortFlag(value) {
|
|
3687
|
+
if (value == null || value === false) {
|
|
3688
|
+
return 0;
|
|
3689
|
+
}
|
|
3690
|
+
const raw = String(value);
|
|
3691
|
+
if (!/^\d+$/u.test(raw)) {
|
|
3692
|
+
throw new UsageError("--port must be an integer from 0 to 65535.");
|
|
3693
|
+
}
|
|
3694
|
+
const port = Number(raw);
|
|
3695
|
+
if (!Number.isSafeInteger(port) || port < 0 || port > 65535) {
|
|
3696
|
+
throw new UsageError("--port must be an integer from 0 to 65535.");
|
|
3697
|
+
}
|
|
3698
|
+
return port;
|
|
3699
|
+
}
|
|
3700
|
+
function formatMetricSummary(metrics, options = {}) {
|
|
3701
|
+
const entries = Object.entries(metrics ?? {}).filter((entry) => Number.isFinite(entry[1]));
|
|
3702
|
+
if (entries.length === 0) {
|
|
3703
|
+
return "n/a";
|
|
3704
|
+
}
|
|
3705
|
+
const limit = options.limit ?? 2;
|
|
3706
|
+
const shown = Number.isFinite(limit)
|
|
3707
|
+
? entries.slice(0, Math.max(0, limit))
|
|
3708
|
+
: entries;
|
|
3709
|
+
const suffix = shown.length < entries.length ? ` (+${entries.length - shown.length})` : "";
|
|
3710
|
+
return `${shown.map(([key, value]) => `${key}: ${formatMetricValue(value)}`).join(", ")}${suffix}`;
|
|
3711
|
+
}
|
|
3712
|
+
function formatMetricValue(value) {
|
|
3713
|
+
if (!Number.isFinite(value)) {
|
|
3714
|
+
return String(value);
|
|
3715
|
+
}
|
|
3716
|
+
if (Number.isInteger(value)) {
|
|
3717
|
+
return String(value);
|
|
3718
|
+
}
|
|
3719
|
+
return value.toFixed(2);
|
|
3720
|
+
}
|
|
3721
|
+
function resolveDir(parsed, positionalDir) {
|
|
3722
|
+
const resolved = path.resolve(asOptionalString(parsed.flags.dir) ?? positionalDir ?? process.cwd());
|
|
3723
|
+
return isWorkbenchSourceYamlPath(resolved) ? path.dirname(resolved) : resolved;
|
|
3724
|
+
}
|
|
3725
|
+
function resolveSourceDir(parsed) {
|
|
3726
|
+
if (parsed.positionals.length > 1) {
|
|
3727
|
+
throw new UsageError("Expected at most one source file or directory argument.");
|
|
3728
|
+
}
|
|
3729
|
+
if (parsed.positionals.length > 0 && parsed.flags.dir !== undefined) {
|
|
3730
|
+
throw new UsageError("Use either --dir or SOURCE, not both.");
|
|
3731
|
+
}
|
|
3732
|
+
return path.resolve(asOptionalString(parsed.flags.dir) ?? parsed.positionals[0] ?? process.cwd());
|
|
3733
|
+
}
|
|
3734
|
+
function isWorkbenchSourceYamlPath(filePath) {
|
|
3735
|
+
return path.basename(filePath) === WORKBENCH_BENCHMARK_FILE;
|
|
3736
|
+
}
|
|
3737
|
+
function readSubjectIdFlag(parsed, snapshot) {
|
|
3738
|
+
const explicit = asOptionalString(parsed.flags.subject) ?? asOptionalString(parsed.flags.subject);
|
|
3739
|
+
if (explicit) {
|
|
3740
|
+
return explicit;
|
|
3741
|
+
}
|
|
3742
|
+
if (snapshot.activeId) {
|
|
3743
|
+
return snapshot.activeId;
|
|
3744
|
+
}
|
|
3745
|
+
throw new UsageError("Missing required --subject; no active subject exists.");
|
|
3746
|
+
}
|
|
3747
|
+
function readPreviewMode(parsed) {
|
|
3748
|
+
const view = asOptionalString(parsed.flags.view) ?? "rendered";
|
|
3749
|
+
if (view !== "diff" && view !== "raw" && view !== "rendered") {
|
|
3750
|
+
throw new UsageError("--view must be diff, raw, or rendered.");
|
|
3751
|
+
}
|
|
3752
|
+
return view;
|
|
3753
|
+
}
|
|
3754
|
+
async function readLocalSpecIfValid(workspace) {
|
|
3755
|
+
try {
|
|
3756
|
+
return (await readLocalProjectSource(workspace)).spec;
|
|
3757
|
+
}
|
|
3758
|
+
catch (error) {
|
|
3759
|
+
if (error.code === "ENOENT" ||
|
|
3760
|
+
error instanceof WorkspaceSnapshotError) {
|
|
3761
|
+
return null;
|
|
3762
|
+
}
|
|
3763
|
+
throw error;
|
|
3764
|
+
}
|
|
3765
|
+
}
|
|
3766
|
+
async function resolveLocalProjectForExecution(workspace, source) {
|
|
3767
|
+
const spec = resolveWorkbenchResolvedSourceYaml(source);
|
|
3768
|
+
const adapters = await resolveWorkbenchAdaptersForProject(workspace, spec);
|
|
3769
|
+
const adapterManifests = adapters.map((adapter) => adapter.manifest);
|
|
3770
|
+
return {
|
|
3771
|
+
spec: applyDefaultWorkbenchAdapterAuthProfiles(spec, adapterManifests),
|
|
3772
|
+
adapterManifests,
|
|
3773
|
+
};
|
|
3774
|
+
}
|
|
3775
|
+
function completedJobOutputFiles(job) {
|
|
3776
|
+
const output = asJsonRecord(job.output);
|
|
3777
|
+
const files = Array.isArray(output.files)
|
|
3778
|
+
? output.files.filter(isSurfaceSnapshotFile)
|
|
3779
|
+
: [];
|
|
3780
|
+
return normalizeSurfaceFiles(files);
|
|
3781
|
+
}
|
|
3782
|
+
function asJsonRecord(value) {
|
|
3783
|
+
return value && typeof value === "object" && !Array.isArray(value)
|
|
3784
|
+
? value
|
|
3785
|
+
: {};
|
|
3786
|
+
}
|
|
3787
|
+
function isSurfaceSnapshotFile(value) {
|
|
3788
|
+
const record = asJsonRecord(value);
|
|
3789
|
+
return (typeof record.path === "string" &&
|
|
3790
|
+
typeof record.content === "string" &&
|
|
3791
|
+
(record.kind === undefined ||
|
|
3792
|
+
record.kind === "text" ||
|
|
3793
|
+
record.kind === "binary") &&
|
|
3794
|
+
(record.encoding === undefined ||
|
|
3795
|
+
record.encoding === "utf8" ||
|
|
3796
|
+
record.encoding === "base64"));
|
|
3797
|
+
}
|
|
3798
|
+
function createLocalEvent(type, at, event) {
|
|
3799
|
+
return {
|
|
3800
|
+
id: `evt_${Math.random().toString(36).slice(2, 10)}_${Date.now().toString(36)}`,
|
|
3801
|
+
at,
|
|
3802
|
+
type,
|
|
3803
|
+
...event,
|
|
3804
|
+
};
|
|
3805
|
+
}
|
|
3806
|
+
async function writeFileIfMissing(filePath, content) {
|
|
3807
|
+
try {
|
|
3808
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
3809
|
+
await fs.writeFile(filePath, content, { flag: "wx" });
|
|
3810
|
+
}
|
|
3811
|
+
catch (error) {
|
|
3812
|
+
if (error.code !== "EEXIST") {
|
|
3813
|
+
throw error;
|
|
3814
|
+
}
|
|
3815
|
+
}
|
|
3816
|
+
}
|
|
3817
|
+
async function copyInitSeedIfProvided(parsed, workspace, seed) {
|
|
3818
|
+
const from = asOptionalString(parsed.flags.from);
|
|
3819
|
+
if (!from) {
|
|
3820
|
+
return;
|
|
3821
|
+
}
|
|
3822
|
+
const source = path.resolve(from);
|
|
3823
|
+
const stats = await fs.stat(source).catch((error) => {
|
|
3824
|
+
throw new UsageError(`--from path does not exist: ${error.path ?? source}`);
|
|
3825
|
+
});
|
|
3826
|
+
if (stats.isDirectory()) {
|
|
3827
|
+
const target = path.join(workspace, seed.directoryTarget);
|
|
3828
|
+
await fs.mkdir(target, { recursive: true });
|
|
3829
|
+
await fs.cp(source, target, {
|
|
3830
|
+
recursive: true,
|
|
3831
|
+
force: false,
|
|
3832
|
+
errorOnExist: false,
|
|
3833
|
+
});
|
|
3834
|
+
return;
|
|
3835
|
+
}
|
|
3836
|
+
if (!stats.isFile()) {
|
|
3837
|
+
throw new UsageError("--from must point to a file or directory.");
|
|
3838
|
+
}
|
|
3839
|
+
const target = path.join(workspace, seed.fileTarget);
|
|
3840
|
+
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
3841
|
+
await fs
|
|
3842
|
+
.copyFile(source, target, fs.constants.COPYFILE_EXCL)
|
|
3843
|
+
.catch((error) => {
|
|
3844
|
+
if (error.code !== "EEXIST") {
|
|
3845
|
+
throw error;
|
|
3846
|
+
}
|
|
3847
|
+
});
|
|
3848
|
+
}
|
|
3849
|
+
function formatSpecOptimizer(spec) {
|
|
3850
|
+
return spec.improve ? `adapter:${spec.improve.use}` : "optimizer not configured";
|
|
3851
|
+
}
|
|
3852
|
+
async function writeFiles(outputDir, files) {
|
|
3853
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
3854
|
+
for (const file of files) {
|
|
3855
|
+
const targetPath = safeOutputPath(outputDir, file.path);
|
|
3856
|
+
await fs.mkdir(path.dirname(targetPath), { recursive: true });
|
|
3857
|
+
await fs.writeFile(targetPath, file.encoding === "base64"
|
|
3858
|
+
? Buffer.from(file.content, "base64")
|
|
3859
|
+
: file.content);
|
|
3860
|
+
await fs.chmod(targetPath, file.executable === true ? 0o755 : 0o644);
|
|
3861
|
+
}
|
|
3862
|
+
}
|
|
3863
|
+
async function syncSourceFiles(outputDir, files) {
|
|
3864
|
+
const previousPaths = await readManagedSourceFilePaths(outputDir);
|
|
3865
|
+
const nextPaths = new Set(files.map((file) => file.path));
|
|
3866
|
+
for (const previousPath of previousPaths) {
|
|
3867
|
+
if (nextPaths.has(previousPath)) {
|
|
3868
|
+
continue;
|
|
3869
|
+
}
|
|
3870
|
+
await fs.rm(safeOutputPath(outputDir, previousPath), { force: true });
|
|
3871
|
+
await removeEmptyParents(outputDir, path.dirname(previousPath));
|
|
3872
|
+
}
|
|
3873
|
+
await writeFiles(outputDir, files);
|
|
3874
|
+
}
|
|
3875
|
+
async function readManagedSourceFilePaths(outputDir) {
|
|
3876
|
+
try {
|
|
3877
|
+
const source = await readLocalProjectSource(outputDir);
|
|
3878
|
+
return new Set(source.sourceFiles.map((file) => file.path));
|
|
3879
|
+
}
|
|
3880
|
+
catch (error) {
|
|
3881
|
+
if (error.code === "ENOENT" ||
|
|
3882
|
+
error instanceof WorkspaceSnapshotError) {
|
|
3883
|
+
return new Set();
|
|
3884
|
+
}
|
|
3885
|
+
throw error;
|
|
3886
|
+
}
|
|
3887
|
+
}
|
|
3888
|
+
async function removeEmptyParents(outputDir, relativeDir) {
|
|
3889
|
+
let current = path.normalize(relativeDir);
|
|
3890
|
+
while (current && current !== "." && current !== path.sep) {
|
|
3891
|
+
const absolute = safeOutputPath(outputDir, current);
|
|
3892
|
+
try {
|
|
3893
|
+
await fs.rmdir(absolute);
|
|
3894
|
+
}
|
|
3895
|
+
catch {
|
|
3896
|
+
return;
|
|
3897
|
+
}
|
|
3898
|
+
current = path.dirname(current);
|
|
3899
|
+
}
|
|
3900
|
+
}
|
|
3901
|
+
function safeOutputPath(outputDir, relativePath) {
|
|
3902
|
+
const targetPath = path.resolve(outputDir, relativePath);
|
|
3903
|
+
const root = path.resolve(outputDir);
|
|
3904
|
+
if (targetPath !== root && !targetPath.startsWith(`${root}${path.sep}`)) {
|
|
3905
|
+
throw new UsageError(`Unsafe export path: ${relativePath}`);
|
|
3906
|
+
}
|
|
3907
|
+
return targetPath;
|
|
3908
|
+
}
|
|
3909
|
+
function writeOutput(value, parsed, io, formatText) {
|
|
3910
|
+
if (parsed.flags.json === true) {
|
|
3911
|
+
writeJson(value, io);
|
|
3912
|
+
}
|
|
3913
|
+
else {
|
|
3914
|
+
io.stdout.write(`${formatText(value)}\n`);
|
|
3915
|
+
}
|
|
3916
|
+
}
|
|
3917
|
+
function writeJson(value, io) {
|
|
3918
|
+
io.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
|
|
3919
|
+
}
|
|
3920
|
+
async function loadConfig() {
|
|
3921
|
+
try {
|
|
3922
|
+
return JSON.parse(await fs.readFile(configPath(), "utf8"));
|
|
3923
|
+
}
|
|
3924
|
+
catch (error) {
|
|
3925
|
+
if (error.code === "ENOENT") {
|
|
3926
|
+
return {};
|
|
3927
|
+
}
|
|
3928
|
+
throw error;
|
|
3929
|
+
}
|
|
3930
|
+
}
|
|
3931
|
+
async function writeConfig(config) {
|
|
3932
|
+
const filePath = configPath();
|
|
3933
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
3934
|
+
await fs.writeFile(filePath, `${JSON.stringify(config, null, 2)}\n`);
|
|
3935
|
+
}
|
|
3936
|
+
function configPath() {
|
|
3937
|
+
return path.join(os.homedir(), ".workbench", "workbench.json");
|
|
3938
|
+
}
|
|
3939
|
+
function normalizeBaseUrl(value) {
|
|
3940
|
+
return value.replace(/\/+$/u, "");
|
|
3941
|
+
}
|
|
3942
|
+
class UsageError extends Error {
|
|
3943
|
+
}
|