@workbench-ai/workbench 0.0.54 → 0.0.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/command-model.js +1 -1
- package/dist/index.js +55 -29
- package/package.json +4 -4
package/dist/command-model.js
CHANGED
|
@@ -140,7 +140,7 @@ const commandHelp = Object.fromEntries(Object.entries({
|
|
|
140
140
|
" workbench improve [SOURCE] [--dir DIR] [--from CANDIDATE_ID] [--runs RUN] [--budget N] [--samples N] [--rerun] [--json]",
|
|
141
141
|
" workbench improve --hosted [SOURCE] [--dir DIR] [--benchmark OWNER/BENCHMARK] [--base CANDIDATE_ID] [--runs RUN] [--budget N] [--samples N] [--rerun] [--watch] [--dry-run] [--json]",
|
|
142
142
|
"",
|
|
143
|
-
"Ensure a candidate improvement exists for the selected base, run, budget, and samples. Without --hosted, execution writes local records. With --hosted, Workbench starts or reuses hosted work against the configured remote or --benchmark target. Completed work is reused only when base, run configuration, source, adapters, benchmark, budget, and samples match; use --rerun to intentionally spend again.",
|
|
143
|
+
"Ensure a candidate improvement exists for the selected base, run, budget, and samples. Improve defaults to the evaluated active candidate when it belongs to the current benchmark fingerprint; otherwise it evaluates and uses the authored current candidate. Without --hosted, execution writes local records. With --hosted, Workbench starts or reuses hosted work against the configured remote or --benchmark target. Completed work is reused only when base, run configuration, source, adapters, benchmark, budget, and samples match; use --rerun to intentionally spend again.",
|
|
144
144
|
"",
|
|
145
145
|
"Examples:",
|
|
146
146
|
" workbench improve --budget 1 --samples 1",
|
package/dist/index.js
CHANGED
|
@@ -993,35 +993,30 @@ async function localRun(argv, io, runtimeOptions) {
|
|
|
993
993
|
async function ensureLocalImproveBaseCandidate(args) {
|
|
994
994
|
let snapshot = await loadLocalArchive(args.workspace);
|
|
995
995
|
const explicitBase = asOptionalString(args.parsed.flags.from);
|
|
996
|
-
const benchmarkFingerprint =
|
|
996
|
+
const benchmarkFingerprint = localBenchmarkFingerprint(args.projectSource);
|
|
997
|
+
const baseCandidateArgs = {
|
|
998
|
+
workspace: args.workspace,
|
|
999
|
+
benchmarkFingerprint,
|
|
1000
|
+
projectSource: args.projectSource,
|
|
1001
|
+
samples: args.samples,
|
|
1002
|
+
rerun: args.parsed.flags.rerun === true,
|
|
1003
|
+
io: args.io,
|
|
1004
|
+
runtimeOptions: args.runtimeOptions,
|
|
1005
|
+
};
|
|
997
1006
|
if (explicitBase) {
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
}
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
if (
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
explicitBase,
|
|
1011
|
-
"--runs",
|
|
1012
|
-
args.projectSource.spec.candidate.selectedRunId,
|
|
1013
|
-
"--samples",
|
|
1014
|
-
String(args.samples),
|
|
1015
|
-
...(args.parsed.flags.rerun === true ? ["--rerun"] : []),
|
|
1016
|
-
"--json",
|
|
1017
|
-
], createSilentIo(args.io), args.runtimeOptions);
|
|
1018
|
-
if (code !== 0) {
|
|
1019
|
-
throw new UsageError(`Base candidate ${explicitBase} eval failed; improve was not started.`);
|
|
1020
|
-
}
|
|
1021
|
-
snapshot = await loadLocalArchive(args.workspace);
|
|
1022
|
-
candidate = readLocalCandidate(snapshot, explicitBase);
|
|
1007
|
+
return await ensureEvaluatedLocalImproveBaseCandidate({
|
|
1008
|
+
...baseCandidateArgs,
|
|
1009
|
+
candidateId: explicitBase,
|
|
1010
|
+
});
|
|
1011
|
+
}
|
|
1012
|
+
if (snapshot.activeId) {
|
|
1013
|
+
const activeCandidate = readLocalCandidate(snapshot, snapshot.activeId);
|
|
1014
|
+
if (activeCandidate.benchmarkFingerprint === benchmarkFingerprint) {
|
|
1015
|
+
return await ensureEvaluatedLocalImproveBaseCandidate({
|
|
1016
|
+
...baseCandidateArgs,
|
|
1017
|
+
candidateId: activeCandidate.id,
|
|
1018
|
+
});
|
|
1023
1019
|
}
|
|
1024
|
-
return candidate;
|
|
1025
1020
|
}
|
|
1026
1021
|
const candidateFingerprint = localCandidateFingerprint(args.projectSource);
|
|
1027
1022
|
const existing = snapshot.candidates.find((candidate) => candidate.benchmarkFingerprint === benchmarkFingerprint &&
|
|
@@ -1063,6 +1058,36 @@ async function ensureLocalImproveBaseCandidate(args) {
|
|
|
1063
1058
|
}
|
|
1064
1059
|
return evaluated;
|
|
1065
1060
|
}
|
|
1061
|
+
async function ensureEvaluatedLocalImproveBaseCandidate(args) {
|
|
1062
|
+
let snapshot = await loadLocalArchive(args.workspace);
|
|
1063
|
+
let candidate = readLocalCandidate(snapshot, args.candidateId);
|
|
1064
|
+
if (candidate.benchmarkFingerprint !== args.benchmarkFingerprint) {
|
|
1065
|
+
throw new UsageError(`Base candidate ${args.candidateId} belongs to benchmark ${candidate.benchmarkFingerprint}, not ${args.benchmarkFingerprint}.`);
|
|
1066
|
+
}
|
|
1067
|
+
if (!candidate.candidateFingerprint) {
|
|
1068
|
+
throw new UsageError(`Base candidate ${args.candidateId} is missing a candidate fingerprint.`);
|
|
1069
|
+
}
|
|
1070
|
+
if (candidate.status === "evaluated" || candidate.eval) {
|
|
1071
|
+
return candidate;
|
|
1072
|
+
}
|
|
1073
|
+
const code = await localEvaluateCandidate([
|
|
1074
|
+
"--dir",
|
|
1075
|
+
args.workspace,
|
|
1076
|
+
"--candidate",
|
|
1077
|
+
args.candidateId,
|
|
1078
|
+
"--runs",
|
|
1079
|
+
args.projectSource.spec.candidate.selectedRunId,
|
|
1080
|
+
"--samples",
|
|
1081
|
+
String(args.samples),
|
|
1082
|
+
...(args.rerun ? ["--rerun"] : []),
|
|
1083
|
+
"--json",
|
|
1084
|
+
], createSilentIo(args.io), args.runtimeOptions);
|
|
1085
|
+
if (code !== 0) {
|
|
1086
|
+
throw new UsageError(`Base candidate ${args.candidateId} eval failed; improve was not started.`);
|
|
1087
|
+
}
|
|
1088
|
+
snapshot = await loadLocalArchive(args.workspace);
|
|
1089
|
+
return readLocalCandidate(snapshot, args.candidateId);
|
|
1090
|
+
}
|
|
1066
1091
|
function createSilentIo(io) {
|
|
1067
1092
|
const sink = new class extends Writable {
|
|
1068
1093
|
_write(_chunk, _encoding, callback) {
|
|
@@ -2922,6 +2947,7 @@ async function pushBenchmark(argv, io) {
|
|
|
2922
2947
|
const runtime = await exportLocalRuntimeBundle(dir, {
|
|
2923
2948
|
currentBenchmarkFingerprint: localBenchmarkFingerprint(source),
|
|
2924
2949
|
});
|
|
2950
|
+
const localRuntimeFingerprint = workbenchRuntimeBundleFingerprint(runtime);
|
|
2925
2951
|
const state = localProjectState({
|
|
2926
2952
|
source,
|
|
2927
2953
|
runtime,
|
|
@@ -2941,7 +2967,7 @@ async function pushBenchmark(argv, io) {
|
|
|
2941
2967
|
sourceFileCount: sourceFileCount(source),
|
|
2942
2968
|
runtime: runtimeBundleStats(runtime),
|
|
2943
2969
|
sourceFingerprint: state.source.fingerprint,
|
|
2944
|
-
runtimeFingerprint:
|
|
2970
|
+
runtimeFingerprint: localRuntimeFingerprint,
|
|
2945
2971
|
}, parsed, io, () => `Would push benchmark ${source.spec.name}.`);
|
|
2946
2972
|
return 0;
|
|
2947
2973
|
}
|
|
@@ -2990,7 +3016,7 @@ async function pushBenchmark(argv, io) {
|
|
|
2990
3016
|
sourceFileCount: sourceFileCount(source),
|
|
2991
3017
|
runtime: runtimeBundleStats(runtime),
|
|
2992
3018
|
sourceFingerprint: state.source.fingerprint,
|
|
2993
|
-
runtimeFingerprint:
|
|
3019
|
+
runtimeFingerprint: localRuntimeFingerprint,
|
|
2994
3020
|
}, parsed, io, () => `Would push ${sourceFileCount(source)} source file(s) and runtime history to ${origin.remote}.`);
|
|
2995
3021
|
return 0;
|
|
2996
3022
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@workbench-ai/workbench",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.55",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -21,9 +21,9 @@
|
|
|
21
21
|
],
|
|
22
22
|
"dependencies": {
|
|
23
23
|
"yaml": "^2.8.2",
|
|
24
|
-
"@workbench-ai/workbench-
|
|
25
|
-
"@workbench-ai/workbench-core": "0.0.
|
|
26
|
-
"@workbench-ai/workbench-
|
|
24
|
+
"@workbench-ai/workbench-protocol": "0.0.55",
|
|
25
|
+
"@workbench-ai/workbench-core": "0.0.55",
|
|
26
|
+
"@workbench-ai/workbench-built-in-adapters": "0.0.55"
|
|
27
27
|
},
|
|
28
28
|
"devDependencies": {
|
|
29
29
|
"@tailwindcss/postcss": "^4.2.2",
|