@workbench-ai/workbench 0.0.51 → 0.0.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark-fingerprint.d.ts +1 -3
- package/dist/benchmark-fingerprint.d.ts.map +1 -1
- package/dist/benchmark-fingerprint.js +19 -87
- package/dist/command-model.d.ts.map +1 -1
- package/dist/command-model.js +5 -3
- package/dist/dev-open/client.js +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +97 -77
- package/dist/local-archive.d.ts +1 -1
- package/dist/local-archive.d.ts.map +1 -1
- package/dist/local-archive.js +111 -18
- package/dist/project-source.d.ts +1 -0
- package/dist/project-source.d.ts.map +1 -1
- package/dist/project-source.js +23 -1
- package/package.json +4 -4
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA0IA,UAAU,KAAK;IACb,KAAK,EAAE,MAAM,CAAC,cAAc,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;CAC/B;AA4BD,UAAU,iBAAiB;CAAG;AA4K9B,wBAAsB,MAAM,CAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,GAAE,KAIH,EACD,cAAc,GAAE,iBAAsB,GACrC,OAAO,CAAC,MAAM,CAAC,CAmHjB"}
|
package/dist/index.js
CHANGED
|
@@ -6,7 +6,7 @@ import os from "node:os";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { Writable } from "node:stream";
|
|
8
8
|
import { createCandidateFilePreview, createBaselineCandidateJob as createRuntimeBaselineCandidateJob, evaluationScorecardId, evaluationMeanMetrics, executeWorkbenchExecutionJob, engineResolveBindingForSpec, filterOptimizerTraceJobsForCaseIds, filterCandidateSourceFiles, formatWorkbenchCaseSelector, formatWorkbenchSelectionPolicy, workbenchCaseSelectorUsesAllCases, workbenchExecutionPurpose, workbenchRunExecutionFingerprint, createWorkbenchAdapterAuthBundle, createOptimizerTraceInputFiles, DOCKER_SANDBOX_BACKEND, localWorkbenchAdapterAuthStore, materializeWorkbenchRunResult, normalizeSurfaceFiles, planWorkbenchExecutionJobsForPurpose, runWorkbenchExecutionDag, resolveEngineCaseExecutionConfig, resolveWorkbenchResolvedSourceYaml, summarizeCandidateFiles, validateWorkbenchRunEnvelope, parseWorkbenchAdapterAuthTarget, workbenchEngineCaseIdsForImproveEvaluation, workbenchEngineCaseIdsForSelector, workbenchImproveOptimizeSelector, workbenchImproveSelectionPolicy, workbenchProjectSourceFingerprint, workbenchRuntimeBundleFingerprint, } from "@workbench-ai/workbench-core";
|
|
9
|
-
import { assertWorkbenchAdapterOperationResultOk, collectWorkbenchAdapterAuthRequirements,
|
|
9
|
+
import { assertWorkbenchAdapterOperationResultOk, collectWorkbenchAdapterAuthRequirements, normalizeWorkbenchAdapterOperationRequest, readWorkbenchAdapterOperationResult, workbenchAdapterOperationCommand, workbenchAdapterOperationResultPath, withDefaultWorkbenchAdapterAuthProfiles as applyDefaultWorkbenchAdapterAuthProfiles, } from "@workbench-ai/workbench-protocol";
|
|
10
10
|
import { builtinLocalTraceAdapter, builtinLocalTraceAdapters, sortLocalTraceRefs, } from "@workbench-ai/workbench-built-in-adapters/local-traces";
|
|
11
11
|
import { commandUsage, HOSTED_WATCH_LIFECYCLE_NOTE, LOCAL_DEV_OPEN_LIFECYCLE_NOTE, rootUsage, } from "./command-model.js";
|
|
12
12
|
import { startLocalWorkbenchDevServer } from "./dev-open-server.js";
|
|
@@ -15,7 +15,7 @@ import { defaultAdapterManifests, composeRuntimeDockerfileWithAdapters, resolveD
|
|
|
15
15
|
import { createAdapterCommandEnv } from "./adapter-command-env.js";
|
|
16
16
|
import { loadLocalArchive, loadLocalArchiveIndex, exportLocalRuntimeBundle, importLocalRuntimeBundle, runtimeBundleStats, materializeCandidateRoot, readLocalCandidate, readLocalCandidateFiles, readLocalJobs, saveLocalArchive, saveLocalJobs, setLocalActive, upsertLocalRun, upsertLocalCandidate, upsertLocalEvaluation, } from "./local-archive.js";
|
|
17
17
|
import { WorkspaceSnapshotError, } from "./workspace-snapshot.js";
|
|
18
|
-
import { readLocalProjectSource, WORKBENCH_BENCHMARK_FILE, } from "./project-source.js";
|
|
18
|
+
import { hostedEngineResolveFiles, readLocalProjectSource, WORKBENCH_BENCHMARK_FILE, } from "./project-source.js";
|
|
19
19
|
import { localBenchmarkFingerprint, localCandidateFingerprint, } from "./benchmark-fingerprint.js";
|
|
20
20
|
const require = createRequire(import.meta.url);
|
|
21
21
|
function getCliVersion() {
|
|
@@ -869,14 +869,15 @@ async function localRun(argv, io, runtimeOptions) {
|
|
|
869
869
|
},
|
|
870
870
|
});
|
|
871
871
|
for (const candidate of materialized.candidates) {
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
872
|
+
const localCandidate = localCandidateRecord(candidate);
|
|
873
|
+
outputCandidateId = localCandidate.id;
|
|
874
|
+
snapshot = upsertLocalCandidate(snapshot, localCandidate, materialized.candidateFiles[localCandidate.id] ?? []);
|
|
875
|
+
events.push(createLocalEvent("candidate_created", localCandidate.createdAt, {
|
|
875
876
|
runId,
|
|
876
|
-
candidateId:
|
|
877
|
-
baseId:
|
|
878
|
-
status:
|
|
879
|
-
metrics: evaluationMeanMetrics(
|
|
877
|
+
candidateId: localCandidate.id,
|
|
878
|
+
baseId: localCandidate.baseId,
|
|
879
|
+
status: localCandidate.status,
|
|
880
|
+
metrics: evaluationMeanMetrics(localCandidate.eval),
|
|
880
881
|
}));
|
|
881
882
|
}
|
|
882
883
|
for (const evaluation of materialized.evaluations) {
|
|
@@ -1280,7 +1281,7 @@ async function localEvaluateCandidate(argv, io, runtimeOptions) {
|
|
|
1280
1281
|
previousCandidate: existingCandidate ?? null,
|
|
1281
1282
|
existingCandidateCount: snapshot.candidates.length,
|
|
1282
1283
|
});
|
|
1283
|
-
for (const candidateRecord of materialized.candidates) {
|
|
1284
|
+
for (const candidateRecord of materialized.candidates.map(localCandidateRecord)) {
|
|
1284
1285
|
snapshot = upsertLocalCandidate(snapshot, candidateRecord, materialized.candidateFiles[candidateRecord.id] ?? []);
|
|
1285
1286
|
}
|
|
1286
1287
|
if (materialized.activeCandidateId) {
|
|
@@ -3005,8 +3006,6 @@ async function pushBenchmark(argv, io) {
|
|
|
3005
3006
|
const nextOrigin = await writeWorkbenchOriginFromState(dir, {
|
|
3006
3007
|
baseUrl,
|
|
3007
3008
|
state: response.state,
|
|
3008
|
-
project: publishedProject,
|
|
3009
|
-
sourceFingerprint: state.source.fingerprint,
|
|
3010
3009
|
});
|
|
3011
3010
|
writeOutput({
|
|
3012
3011
|
ok: true,
|
|
@@ -3040,8 +3039,6 @@ async function createHostedBenchmarkFromState(args) {
|
|
|
3040
3039
|
const origin = await writeWorkbenchOriginFromState(args.dir, {
|
|
3041
3040
|
baseUrl: args.baseUrl,
|
|
3042
3041
|
state: result.state,
|
|
3043
|
-
project,
|
|
3044
|
-
sourceFingerprint: args.state.source.fingerprint,
|
|
3045
3042
|
});
|
|
3046
3043
|
return { project, origin, result };
|
|
3047
3044
|
}
|
|
@@ -3150,11 +3147,11 @@ async function applyProjectStateToLocal(args) {
|
|
|
3150
3147
|
await assertLocalSourceMatchesOrigin(args.dir, args.origin);
|
|
3151
3148
|
}
|
|
3152
3149
|
await syncSourceFiles(args.dir, args.state.source.files);
|
|
3153
|
-
const
|
|
3150
|
+
const benchmarkFingerprint = localBenchmarkFingerprint(await readLocalProjectSource(args.dir));
|
|
3151
|
+
const runtimeImport = await importLocalRuntimeBundle(args.dir, args.state.runtime, benchmarkFingerprint);
|
|
3154
3152
|
const origin = await writeWorkbenchOriginFromState(args.dir, {
|
|
3155
3153
|
baseUrl: args.baseUrl,
|
|
3156
3154
|
state: args.state,
|
|
3157
|
-
sourceFingerprint: await localSourceFingerprint(args.dir),
|
|
3158
3155
|
});
|
|
3159
3156
|
return {
|
|
3160
3157
|
origin,
|
|
@@ -3193,20 +3190,21 @@ async function retryHostedWorkflow(argv, io) {
|
|
|
3193
3190
|
method: "POST",
|
|
3194
3191
|
body: retryTarget.request,
|
|
3195
3192
|
}, target.baseUrl);
|
|
3196
|
-
const
|
|
3193
|
+
const runTarget = hostedTargetForRunStartResponse(target, response);
|
|
3194
|
+
const startedRun = withRunUrls(runTarget, response.run);
|
|
3197
3195
|
if (parsed.flags.watch === true) {
|
|
3198
3196
|
if (parsed.flags.json !== true) {
|
|
3199
3197
|
io.stdout.write(`${formatHostedRunStarted(startedRun, retryTarget.workflow).trimEnd()}\n${HOSTED_WATCH_LIFECYCLE_NOTE}\n`);
|
|
3200
3198
|
}
|
|
3201
3199
|
const watched = await watchHostedRun({
|
|
3202
3200
|
parsed,
|
|
3203
|
-
target,
|
|
3201
|
+
target: runTarget,
|
|
3204
3202
|
runId: response.run.id,
|
|
3205
3203
|
intervalMs: watchIntervalMs ?? 1000,
|
|
3206
3204
|
timeoutMs: watchTimeoutMs,
|
|
3207
3205
|
});
|
|
3208
|
-
const outputRun = withRunUrls(
|
|
3209
|
-
await tryImportTerminalHostedProjectState({ target, io });
|
|
3206
|
+
const outputRun = withRunUrls(runTarget, await withHostedRunFailureSummary(runTarget, watched));
|
|
3207
|
+
await tryImportTerminalHostedProjectState({ target: runTarget, io });
|
|
3210
3208
|
const result = {
|
|
3211
3209
|
ok: hostedRunSucceeded(watched),
|
|
3212
3210
|
retried: {
|
|
@@ -3377,12 +3375,10 @@ function hostedRunRecordFailed(run) {
|
|
|
3377
3375
|
}
|
|
3378
3376
|
async function startHostedWorkflow(workflow, argv, io) {
|
|
3379
3377
|
const parsed = parseArgs(argv);
|
|
3380
|
-
|
|
3378
|
+
const allowedFlags = new Set([
|
|
3381
3379
|
"dir",
|
|
3382
3380
|
"benchmark",
|
|
3383
|
-
"base",
|
|
3384
3381
|
"runs",
|
|
3385
|
-
"budget",
|
|
3386
3382
|
"samples",
|
|
3387
3383
|
"rerun",
|
|
3388
3384
|
"watch",
|
|
@@ -3390,7 +3386,15 @@ async function startHostedWorkflow(workflow, argv, io) {
|
|
|
3390
3386
|
"interval-ms",
|
|
3391
3387
|
"timeout-ms",
|
|
3392
3388
|
"json",
|
|
3393
|
-
])
|
|
3389
|
+
]);
|
|
3390
|
+
if (workflow === "eval") {
|
|
3391
|
+
allowedFlags.add("candidate");
|
|
3392
|
+
}
|
|
3393
|
+
else {
|
|
3394
|
+
allowedFlags.add("base");
|
|
3395
|
+
allowedFlags.add("budget");
|
|
3396
|
+
}
|
|
3397
|
+
rejectUnknownFlags(parsed, allowedFlags);
|
|
3394
3398
|
if (parsed.positionals.length > 1) {
|
|
3395
3399
|
throw new UsageError(`workbench ${workflow} --hosted accepts at most one source file or directory argument.`);
|
|
3396
3400
|
}
|
|
@@ -3431,25 +3435,27 @@ async function startHostedWorkflow(workflow, argv, io) {
|
|
|
3431
3435
|
}, parsed, io, () => `Processed ${selectedRunIds.length} hosted candidate run(s); ${failed} failed.`);
|
|
3432
3436
|
return failed === 0 ? 0 : 1;
|
|
3433
3437
|
}
|
|
3434
|
-
const
|
|
3438
|
+
const selectedCandidateId = workflow === "eval"
|
|
3439
|
+
? asOptionalString(parsed.flags.candidate)
|
|
3440
|
+
: asOptionalString(parsed.flags.base);
|
|
3435
3441
|
const request = workflow === "improve"
|
|
3436
3442
|
? {
|
|
3437
3443
|
workflow,
|
|
3438
3444
|
budget,
|
|
3439
3445
|
samples,
|
|
3440
|
-
...(
|
|
3446
|
+
...(selectedCandidateId ? { candidateId: selectedCandidateId } : {}),
|
|
3441
3447
|
}
|
|
3442
3448
|
: {
|
|
3443
3449
|
workflow,
|
|
3444
3450
|
samples,
|
|
3445
|
-
...(
|
|
3451
|
+
...(selectedCandidateId ? { candidateId: selectedCandidateId } : {}),
|
|
3446
3452
|
};
|
|
3447
3453
|
const projectSource = selectedRunIds[0] === defaultProjectSource.candidateRunId
|
|
3448
3454
|
? defaultProjectSource
|
|
3449
3455
|
: await readLocalProjectSource(path.resolve(sourceArg), { runId: selectedRunIds[0] });
|
|
3450
3456
|
request.sourceYaml = projectSource.specSource;
|
|
3451
3457
|
request.adapterFiles = projectSource.adapterFiles;
|
|
3452
|
-
if (workflow === "eval" && !
|
|
3458
|
+
if (workflow === "eval" && !selectedCandidateId) {
|
|
3453
3459
|
request.candidateFiles = projectSource.candidateFiles;
|
|
3454
3460
|
}
|
|
3455
3461
|
if (parsed.flags.rerun === true) {
|
|
@@ -3484,7 +3490,7 @@ async function startHostedWorkflow(workflow, argv, io) {
|
|
|
3484
3490
|
parsed,
|
|
3485
3491
|
target,
|
|
3486
3492
|
samples: request.samples,
|
|
3487
|
-
candidateId:
|
|
3493
|
+
candidateId: selectedCandidateId,
|
|
3488
3494
|
sourceYaml: projectSource.specSource,
|
|
3489
3495
|
adapterFiles: projectSource.adapterFiles,
|
|
3490
3496
|
intervalMs: watchIntervalMs ?? 1000,
|
|
@@ -3496,12 +3502,13 @@ async function startHostedWorkflow(workflow, argv, io) {
|
|
|
3496
3502
|
method: "POST",
|
|
3497
3503
|
body: request,
|
|
3498
3504
|
}, target.baseUrl);
|
|
3499
|
-
const
|
|
3505
|
+
const runTarget = hostedTargetForRunStartResponse(target, response);
|
|
3506
|
+
const startedRun = withRunUrls(runTarget, response.run);
|
|
3500
3507
|
const startedRunOutput = response.reused === true
|
|
3501
3508
|
? { ...startedRun, reused: true }
|
|
3502
3509
|
: startedRun;
|
|
3503
3510
|
if (response.reused === true && response.run.status === "finished") {
|
|
3504
|
-
await tryImportTerminalHostedProjectState({ target, io });
|
|
3511
|
+
await tryImportTerminalHostedProjectState({ target: runTarget, io });
|
|
3505
3512
|
writeOutput({
|
|
3506
3513
|
ok: hostedRunSucceeded(response.run),
|
|
3507
3514
|
reused: true,
|
|
@@ -3517,14 +3524,14 @@ async function startHostedWorkflow(workflow, argv, io) {
|
|
|
3517
3524
|
}
|
|
3518
3525
|
const watched = await watchHostedRun({
|
|
3519
3526
|
parsed,
|
|
3520
|
-
target,
|
|
3527
|
+
target: runTarget,
|
|
3521
3528
|
runId: response.run.id,
|
|
3522
3529
|
intervalMs: watchIntervalMs ?? 1000,
|
|
3523
3530
|
timeoutMs: watchTimeoutMs,
|
|
3524
3531
|
});
|
|
3525
|
-
const outputRun = await withHostedRunFailureSummary(
|
|
3526
|
-
await tryImportTerminalHostedProjectState({ target, io });
|
|
3527
|
-
writeOutput(withRunUrls(
|
|
3532
|
+
const outputRun = await withHostedRunFailureSummary(runTarget, watched);
|
|
3533
|
+
await tryImportTerminalHostedProjectState({ target: runTarget, io });
|
|
3534
|
+
writeOutput(withRunUrls(runTarget, outputRun), parsed, io, formatHostedRunResult);
|
|
3528
3535
|
return hostedRunSucceeded(watched) ? 0 : 1;
|
|
3529
3536
|
}
|
|
3530
3537
|
writeOutput(startedRunOutput, parsed, io, (run) => formatHostedRunStarted(run, workflow).trimEnd());
|
|
@@ -3556,9 +3563,10 @@ async function ensureHostedImproveBaseCandidate(args) {
|
|
|
3556
3563
|
...(args.adapterFiles.length > 0 ? { adapterFiles: args.adapterFiles } : {}),
|
|
3557
3564
|
},
|
|
3558
3565
|
}, args.target.baseUrl);
|
|
3566
|
+
const runTarget = hostedTargetForRunStartResponse(args.target, response);
|
|
3559
3567
|
const watched = await watchHostedRun({
|
|
3560
3568
|
parsed: args.parsed,
|
|
3561
|
-
target:
|
|
3569
|
+
target: runTarget,
|
|
3562
3570
|
runId: response.run.id,
|
|
3563
3571
|
intervalMs: args.intervalMs,
|
|
3564
3572
|
timeoutMs: args.timeoutMs,
|
|
@@ -3569,15 +3577,14 @@ async function ensureHostedImproveBaseCandidate(args) {
|
|
|
3569
3577
|
if (!watched.candidateId) {
|
|
3570
3578
|
throw new UsageError(`Parent candidate eval ${watched.id} did not produce a candidate.`);
|
|
3571
3579
|
}
|
|
3572
|
-
await tryImportTerminalHostedProjectState({ target:
|
|
3580
|
+
await tryImportTerminalHostedProjectState({ target: runTarget, io: args.io });
|
|
3573
3581
|
return watched.candidateId;
|
|
3574
3582
|
}
|
|
3575
3583
|
function hostedWorkflowArgsForRun(args) {
|
|
3576
3584
|
const next = ["--dir", args.sourceDir, "--runs", args.runId, "--json"];
|
|
3577
3585
|
appendStringFlag(next, "benchmark", asOptionalString(args.parsed.flags.benchmark));
|
|
3578
|
-
appendStringFlag(next, "
|
|
3586
|
+
appendStringFlag(next, "candidate", asOptionalString(args.parsed.flags.candidate));
|
|
3579
3587
|
appendStringFlag(next, "samples", asOptionalString(args.parsed.flags.samples));
|
|
3580
|
-
appendStringFlag(next, "budget", asOptionalString(args.parsed.flags.budget));
|
|
3581
3588
|
appendStringFlag(next, "interval-ms", asOptionalString(args.parsed.flags["interval-ms"]));
|
|
3582
3589
|
appendStringFlag(next, "timeout-ms", asOptionalString(args.parsed.flags["timeout-ms"]));
|
|
3583
3590
|
if (args.parsed.flags.watch === true) {
|
|
@@ -3814,6 +3821,9 @@ async function resolveRemoteProject(projectRef, baseUrl) {
|
|
|
3814
3821
|
return response.benchmark;
|
|
3815
3822
|
}
|
|
3816
3823
|
function withRunUrls(target, run) {
|
|
3824
|
+
if (!target.owner || !target.projectName) {
|
|
3825
|
+
return { ...run };
|
|
3826
|
+
}
|
|
3817
3827
|
return {
|
|
3818
3828
|
...run,
|
|
3819
3829
|
urls: buildWorkbenchResourceUrls(target, {
|
|
@@ -3822,6 +3832,31 @@ function withRunUrls(target, run) {
|
|
|
3822
3832
|
}),
|
|
3823
3833
|
};
|
|
3824
3834
|
}
|
|
3835
|
+
function hostedTargetForRunStartResponse(target, response) {
|
|
3836
|
+
const projectId = response.benchmark?.id ?? response.run.projectId ?? target.projectId;
|
|
3837
|
+
if (projectId === target.projectId && !response.benchmark) {
|
|
3838
|
+
return target;
|
|
3839
|
+
}
|
|
3840
|
+
const origin = target.origin?.projectId === projectId ? target.origin : null;
|
|
3841
|
+
const next = {
|
|
3842
|
+
...target,
|
|
3843
|
+
projectId,
|
|
3844
|
+
origin,
|
|
3845
|
+
};
|
|
3846
|
+
if (response.benchmark?.ownerUsername) {
|
|
3847
|
+
next.owner = response.benchmark.ownerUsername;
|
|
3848
|
+
}
|
|
3849
|
+
else {
|
|
3850
|
+
delete next.owner;
|
|
3851
|
+
}
|
|
3852
|
+
if (response.benchmark?.name) {
|
|
3853
|
+
next.projectName = response.benchmark.name;
|
|
3854
|
+
}
|
|
3855
|
+
else {
|
|
3856
|
+
delete next.projectName;
|
|
3857
|
+
}
|
|
3858
|
+
return next;
|
|
3859
|
+
}
|
|
3825
3860
|
function hostedRunEvaluationCandidateId(run, jobs = []) {
|
|
3826
3861
|
if (run.outputCandidateId) {
|
|
3827
3862
|
return run.outputCandidateId;
|
|
@@ -3834,7 +3869,8 @@ function hostedRunEvaluationCandidateId(run, jobs = []) {
|
|
|
3834
3869
|
}
|
|
3835
3870
|
function localProjectState(args) {
|
|
3836
3871
|
const stateSource = localProjectStateSource(args.source);
|
|
3837
|
-
const
|
|
3872
|
+
const runtime = runtimeBundleForProjectVisibility(args.runtime, args.visibility);
|
|
3873
|
+
const runtimeFingerprint = workbenchRuntimeBundleFingerprint(runtime);
|
|
3838
3874
|
return {
|
|
3839
3875
|
schema: "workbench.project.state.v1",
|
|
3840
3876
|
project: {
|
|
@@ -3850,7 +3886,22 @@ function localProjectState(args) {
|
|
|
3850
3886
|
runtimeFingerprint: args.origin?.runtimeFingerprint ?? runtimeFingerprint,
|
|
3851
3887
|
},
|
|
3852
3888
|
source: stateSource,
|
|
3853
|
-
runtime
|
|
3889
|
+
runtime,
|
|
3890
|
+
};
|
|
3891
|
+
}
|
|
3892
|
+
function localCandidateRecord(candidate) {
|
|
3893
|
+
return {
|
|
3894
|
+
...candidate,
|
|
3895
|
+
visibility: "private",
|
|
3896
|
+
};
|
|
3897
|
+
}
|
|
3898
|
+
function runtimeBundleForProjectVisibility(runtime, visibility) {
|
|
3899
|
+
return {
|
|
3900
|
+
...runtime,
|
|
3901
|
+
candidates: runtime.candidates.map((candidate) => ({
|
|
3902
|
+
...candidate,
|
|
3903
|
+
visibility,
|
|
3904
|
+
})),
|
|
3854
3905
|
};
|
|
3855
3906
|
}
|
|
3856
3907
|
function localProjectStateSource(source) {
|
|
@@ -3888,8 +3939,6 @@ function hostedProjectSummaryFromState(state) {
|
|
|
3888
3939
|
ownerUsername: state.project.ownerUsername,
|
|
3889
3940
|
name: state.project.name,
|
|
3890
3941
|
visibility: state.project.visibility,
|
|
3891
|
-
currentSpecVersionId: state.source.revisionId ?? state.base.sourceRevisionId,
|
|
3892
|
-
sourceFingerprint: state.source.fingerprint ?? state.base.sourceFingerprint,
|
|
3893
3942
|
};
|
|
3894
3943
|
}
|
|
3895
3944
|
function sourceFileCount(source) {
|
|
@@ -3910,28 +3959,6 @@ function hostedProjectSourceRequest(source) {
|
|
|
3910
3959
|
resources,
|
|
3911
3960
|
};
|
|
3912
3961
|
}
|
|
3913
|
-
function hostedEngineResolveFiles(source) {
|
|
3914
|
-
return [
|
|
3915
|
-
...source.engineResolveFiles,
|
|
3916
|
-
{
|
|
3917
|
-
path: WORKBENCH_ADAPTER_RESULT_FILE,
|
|
3918
|
-
content: `${JSON.stringify({
|
|
3919
|
-
protocol: WORKBENCH_ADAPTER_RESULT_PROTOCOL,
|
|
3920
|
-
operation: "engine.resolve",
|
|
3921
|
-
ok: true,
|
|
3922
|
-
value: {
|
|
3923
|
-
cases: source.engineCases,
|
|
3924
|
-
...(source.engineResolveEnvironment
|
|
3925
|
-
? { environment: source.engineResolveEnvironment }
|
|
3926
|
-
: {}),
|
|
3927
|
-
},
|
|
3928
|
-
feedback: {
|
|
3929
|
-
path: source.engineResolveFingerprintPath,
|
|
3930
|
-
},
|
|
3931
|
-
}, null, 2)}\n`,
|
|
3932
|
-
},
|
|
3933
|
-
];
|
|
3934
|
-
}
|
|
3935
3962
|
function isRemoteProjectId(value) {
|
|
3936
3963
|
return /^wb_[a-f0-9]{12}$/u.test(value);
|
|
3937
3964
|
}
|
|
@@ -4147,14 +4174,11 @@ async function writeWorkbenchOrigin(dir, input) {
|
|
|
4147
4174
|
return origin;
|
|
4148
4175
|
}
|
|
4149
4176
|
async function writeWorkbenchOriginFromState(dir, args) {
|
|
4150
|
-
const owner = args.
|
|
4151
|
-
const name = args.
|
|
4152
|
-
const sourceRevisionId = args.
|
|
4153
|
-
args.state.source.revisionId ??
|
|
4177
|
+
const owner = args.state.project.ownerUsername;
|
|
4178
|
+
const name = args.state.project.name;
|
|
4179
|
+
const sourceRevisionId = args.state.source.revisionId ??
|
|
4154
4180
|
args.state.base.sourceRevisionId;
|
|
4155
|
-
const sourceFingerprint = args.
|
|
4156
|
-
args.project?.sourceFingerprint ??
|
|
4157
|
-
args.state.source.fingerprint ??
|
|
4181
|
+
const sourceFingerprint = args.state.source.fingerprint ??
|
|
4158
4182
|
args.state.base.sourceFingerprint;
|
|
4159
4183
|
const runtimeFingerprint = args.state.base.runtimeFingerprint ??
|
|
4160
4184
|
workbenchRuntimeBundleFingerprint(args.state.runtime);
|
|
@@ -4164,16 +4188,12 @@ async function writeWorkbenchOriginFromState(dir, args) {
|
|
|
4164
4188
|
return await writeWorkbenchOrigin(dir, {
|
|
4165
4189
|
baseUrl: args.baseUrl,
|
|
4166
4190
|
remote: `${owner}/${name}`,
|
|
4167
|
-
projectId: args.
|
|
4191
|
+
projectId: args.state.project.id,
|
|
4168
4192
|
sourceRevisionId,
|
|
4169
4193
|
sourceFingerprint,
|
|
4170
4194
|
runtimeFingerprint,
|
|
4171
4195
|
});
|
|
4172
4196
|
}
|
|
4173
|
-
async function localSourceFingerprint(dir) {
|
|
4174
|
-
const source = localProjectStateSource(await readLocalProjectSource(dir));
|
|
4175
|
-
return source.fingerprint ?? workbenchProjectSourceFingerprint(source);
|
|
4176
|
-
}
|
|
4177
4197
|
function parseOriginRemote(origin) {
|
|
4178
4198
|
return parseRemoteName(origin.remote);
|
|
4179
4199
|
}
|
package/dist/local-archive.d.ts
CHANGED
|
@@ -24,7 +24,7 @@ export declare function loadLocalArchiveIndex(workspace: string): Promise<LocalA
|
|
|
24
24
|
export declare function saveLocalArchive(workspace: string, snapshot: LocalArchiveSnapshot): Promise<void>;
|
|
25
25
|
export declare function saveLocalJobs(workspace: string, jobs: readonly HostedWorkbenchJob[]): Promise<void>;
|
|
26
26
|
export declare function exportLocalRuntimeBundle(workspace: string): Promise<WorkbenchRuntimeBundle>;
|
|
27
|
-
export declare function importLocalRuntimeBundle(workspace: string, bundle: WorkbenchRuntimeBundle): Promise<WorkbenchRuntimeImportResult>;
|
|
27
|
+
export declare function importLocalRuntimeBundle(workspace: string, bundle: WorkbenchRuntimeBundle, currentBenchmarkFingerprint: string): Promise<WorkbenchRuntimeImportResult>;
|
|
28
28
|
export declare function runtimeBundleStats(bundle: WorkbenchRuntimeBundle): WorkbenchRuntimeBundleStats;
|
|
29
29
|
export declare function sanitizeRuntimeJobForExchange(job: HostedWorkbenchJob): HostedWorkbenchJob;
|
|
30
30
|
export declare function readLocalExecutionFiles(workspace: string, jobId: string): Promise<SurfaceSnapshotFile[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"local-archive.d.ts","sourceRoot":"","sources":["../src/local-archive.ts"],"names":[],"mappings":"AAGA,OAAO,EAQL,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,EAC3B,MAAM,8BAA8B,CAAC;AAOtC,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;IACtD,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG;IAClD,KAAK,CAAC,EAAE,uBAAuB,CAAC;IAChC,aAAa,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACzC,CAAC;AASF,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAevF;AAED,wBAAsB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkBzF;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,kBAAkB,EAAE,GAClC,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,sBAAsB,CAAC,CAuBjC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,sBAAsB,
|
|
1
|
+
{"version":3,"file":"local-archive.d.ts","sourceRoot":"","sources":["../src/local-archive.ts"],"names":[],"mappings":"AAGA,OAAO,EAQL,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,EAC3B,MAAM,8BAA8B,CAAC;AAOtC,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;IACtD,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG;IAClD,KAAK,CAAC,EAAE,uBAAuB,CAAC;IAChC,aAAa,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACzC,CAAC;AASF,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAevF;AAED,wBAAsB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkBzF;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,kBAAkB,EAAE,GAClC,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,sBAAsB,CAAC,CAuBjC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,sBAAsB,EAC9B,2BAA2B,EAAE,MAAM,GAClC,OAAO,CAAC,4BAA4B,CAAC,CAkHvC;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,sBAAsB,GAC7B,2BAA2B,CAE7B;AAED,wBAAgB,6BAA6B,CAC3C,GAAG,EAAE,kBAAkB,GACtB,kBAAkB,CAEpB;AAyDD,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAIhC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,CAAC,CAU1B;AAED,wBAAsB,4BAA4B,CAChD,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAKhC;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,mBAAmB,CAAC,CAU9B;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,UAAU,CAAC,CAUrB;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAM7B;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAE7B;AAED,wBAAsB,iBAAiB,CACrC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAElC;AAED,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,oBAAoB,EAC9B,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,oBAAoB,CAYtB;AAED,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,oBAAoB,EAC9B,UAAU,EAAE,mBAAmB,GAC9B,oBAAoB,CAQtB;AAED,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,oBAAoB,EAC9B,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,SAAS,YAAY,EAAE,GAC9B,oBAAoB,CAYtB;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,oBAAoB,CAK5G;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,GAAG,eAAe,CAMvG;AAED,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,GAAG,mBAAmB,EAAE,CAGlH;AAkoBD,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,OAAO,CAAC,MAAM,EAAE,CAAC,CAOnB;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,mBAAmB,EAAE,EACrC,QAAQ,EAAE,MAAM,GACf,mBAAmB,GAAG,IAAI,CAG5B"}
|
package/dist/local-archive.js
CHANGED
|
@@ -93,7 +93,7 @@ export async function exportLocalRuntimeBundle(workspace) {
|
|
|
93
93
|
events: snapshot.events.map((event) => ({ ...event })),
|
|
94
94
|
};
|
|
95
95
|
}
|
|
96
|
-
export async function importLocalRuntimeBundle(workspace, bundle) {
|
|
96
|
+
export async function importLocalRuntimeBundle(workspace, bundle, currentBenchmarkFingerprint) {
|
|
97
97
|
validateRuntimeBundleSchema(bundle);
|
|
98
98
|
const snapshot = await loadLocalArchive(workspace);
|
|
99
99
|
const existingJobs = (await readLocalJobs(workspace)).map(sanitizeRuntimeJobForExchange);
|
|
@@ -105,7 +105,7 @@ export async function importLocalRuntimeBundle(workspace, bundle) {
|
|
|
105
105
|
const incomingCandidates = bundle.candidates.map(sanitizeWorkbenchRuntimeCandidateForExchange);
|
|
106
106
|
const candidates = mergeRecordsById(existingCandidates, incomingCandidates, (candidate) => candidate.id, (didChange) => {
|
|
107
107
|
changed ||= didChange;
|
|
108
|
-
}).sort(compareLocalCandidateRecords);
|
|
108
|
+
}, runtimeCandidatesCompatibleForExchange, mergeRuntimeCandidateForExchange).sort(compareLocalCandidateRecords);
|
|
109
109
|
const candidateFiles = { ...snapshot.candidateFiles };
|
|
110
110
|
for (const group of bundle.candidateFiles) {
|
|
111
111
|
const candidateId = localRecordName(group.candidateId);
|
|
@@ -121,21 +121,18 @@ export async function importLocalRuntimeBundle(workspace, bundle) {
|
|
|
121
121
|
}
|
|
122
122
|
candidateFiles[candidateId] = files;
|
|
123
123
|
}
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
: snapshot.activeId && candidateIds.has(snapshot.activeId)
|
|
128
|
-
? snapshot.activeId
|
|
129
|
-
: null;
|
|
124
|
+
const activeId = compatibleRuntimeActiveCandidateId(candidates, bundle.activeId ?? null, currentBenchmarkFingerprint) ??
|
|
125
|
+
compatibleRuntimeActiveCandidateId(candidates, snapshot.activeId, currentBenchmarkFingerprint) ??
|
|
126
|
+
latestCompatibleRuntimeCandidateId(candidates, currentBenchmarkFingerprint);
|
|
130
127
|
if (activeId !== snapshot.activeId) {
|
|
131
128
|
changed = true;
|
|
132
129
|
}
|
|
133
130
|
const evaluations = mergeRecordsById(snapshot.evaluations, bundle.evaluations, (evaluation) => evaluation.id, (didChange) => {
|
|
134
131
|
changed ||= didChange;
|
|
135
|
-
}).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
|
|
132
|
+
}, runtimeEvaluationsCompatibleForExchange).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
|
|
136
133
|
const runs = mergeRecordsById(snapshot.runs, bundle.runs, (run) => run.id, (didChange) => {
|
|
137
134
|
changed ||= didChange;
|
|
138
|
-
}).sort((left, right) => left.startedAt.localeCompare(right.startedAt) || left.id.localeCompare(right.id));
|
|
135
|
+
}, runtimeRunsCompatibleForExchange).sort((left, right) => left.startedAt.localeCompare(right.startedAt) || left.id.localeCompare(right.id));
|
|
139
136
|
const events = mergeRecordsById(snapshot.events, bundle.events, runtimeEventKey, (didChange) => {
|
|
140
137
|
changed ||= didChange;
|
|
141
138
|
}).sort((left, right) => left.at.localeCompare(right.at) || left.id.localeCompare(right.id));
|
|
@@ -143,13 +140,20 @@ export async function importLocalRuntimeBundle(workspace, bundle) {
|
|
|
143
140
|
await Promise.all(existingJobs.map(async (job) => {
|
|
144
141
|
executionFilesByJobId.set(job.id, await readLocalExecutionFiles(workspace, job.id));
|
|
145
142
|
}));
|
|
143
|
+
const existingJobById = new Map(existingJobs.map((job) => [job.id, job]));
|
|
144
|
+
const incomingJobById = new Map(bundle.jobs.map(sanitizeRuntimeJobForExchange).map((job) => [job.id, job]));
|
|
146
145
|
for (const group of bundle.executionFiles) {
|
|
147
146
|
const jobId = localRecordName(group.jobId);
|
|
148
147
|
const files = copySurfaceFiles(group.files);
|
|
149
148
|
const existing = executionFilesByJobId.get(jobId);
|
|
150
149
|
if (existing) {
|
|
151
150
|
if (!workbenchSurfaceFilesEqualForExchange(existing, files)) {
|
|
152
|
-
|
|
151
|
+
const existingJob = existingJobById.get(jobId) ?? null;
|
|
152
|
+
const incomingJob = incomingJobById.get(jobId) ?? null;
|
|
153
|
+
if (!existingJob || !incomingJob || !runtimeJobsEqualForExchange(existingJob, incomingJob)) {
|
|
154
|
+
throw new Error(`Runtime history conflict for execution files ${jobId}.`);
|
|
155
|
+
}
|
|
156
|
+
changed = true;
|
|
153
157
|
}
|
|
154
158
|
}
|
|
155
159
|
else {
|
|
@@ -332,7 +336,7 @@ function validateRuntimeBundleSchema(bundle) {
|
|
|
332
336
|
throw new Error("Unsupported Workbench runtime bundle.");
|
|
333
337
|
}
|
|
334
338
|
}
|
|
335
|
-
function mergeRecordsById(existing, incoming, idFor, markChanged, equal = runtimeRecordsEqual) {
|
|
339
|
+
function mergeRecordsById(existing, incoming, idFor, markChanged, equal = runtimeRecordsEqual, merge = (_left, right) => right) {
|
|
336
340
|
const records = new Map();
|
|
337
341
|
for (const record of existing) {
|
|
338
342
|
records.set(localRecordName(idFor(record)), record);
|
|
@@ -340,13 +344,19 @@ function mergeRecordsById(existing, incoming, idFor, markChanged, equal = runtim
|
|
|
340
344
|
for (const record of incoming) {
|
|
341
345
|
const id = localRecordName(idFor(record));
|
|
342
346
|
const previous = records.get(id);
|
|
343
|
-
if (!previous
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
+
if (!previous) {
|
|
348
|
+
markChanged(true);
|
|
349
|
+
records.set(id, record);
|
|
350
|
+
continue;
|
|
351
|
+
}
|
|
352
|
+
if (!equal(previous, record)) {
|
|
353
|
+
throw new Error(`Runtime history conflict for id ${id}.`);
|
|
354
|
+
}
|
|
355
|
+
const merged = merge(previous, record);
|
|
356
|
+
if (!runtimeRecordsEqual(previous, merged)) {
|
|
347
357
|
markChanged(true);
|
|
348
358
|
}
|
|
349
|
-
records.set(id,
|
|
359
|
+
records.set(id, merged);
|
|
350
360
|
}
|
|
351
361
|
return [...records.values()];
|
|
352
362
|
}
|
|
@@ -355,7 +365,10 @@ function runtimeRecordsEqual(left, right) {
|
|
|
355
365
|
JSON.stringify(canonicalRuntimeJson(right));
|
|
356
366
|
}
|
|
357
367
|
function runtimeJobsEqualForExchange(left, right) {
|
|
358
|
-
|
|
368
|
+
if (runtimeRecordsEqual(runtimeComparableJob(left), runtimeComparableJob(right))) {
|
|
369
|
+
return true;
|
|
370
|
+
}
|
|
371
|
+
return runtimeRecordsEqual(runtimeJobIdentityForExchange(left), runtimeJobIdentityForExchange(right));
|
|
359
372
|
}
|
|
360
373
|
function runtimeComparableJob(job) {
|
|
361
374
|
const comparable = sanitizeRuntimeJobForExchange(job);
|
|
@@ -369,6 +382,83 @@ function runtimeComparableJob(job) {
|
|
|
369
382
|
output: portableOutput,
|
|
370
383
|
};
|
|
371
384
|
}
|
|
385
|
+
function runtimeCandidatesCompatibleForExchange(left, right) {
|
|
386
|
+
return runtimeRecordsEqual(runtimeCandidateIdentityForExchange(left), runtimeCandidateIdentityForExchange(right));
|
|
387
|
+
}
|
|
388
|
+
function runtimeCandidateIdentityForExchange(candidate) {
|
|
389
|
+
const { eval: _eval, prompt: _prompt, meta: _meta, status: _status, usage: _usage, visibility: _visibility, ownerUserId: _ownerUserId, ownerUsername: _ownerUsername, metrics: _metrics, candidateRunId: _candidateRunId, candidateRunName: _candidateRunName, ...identity } = candidate;
|
|
390
|
+
return identity;
|
|
391
|
+
}
|
|
392
|
+
function mergeRuntimeCandidateForExchange(left, right) {
|
|
393
|
+
return {
|
|
394
|
+
...left,
|
|
395
|
+
...right,
|
|
396
|
+
...(right.eval ? { eval: right.eval } : left.eval ? { eval: left.eval } : {}),
|
|
397
|
+
...(right.prompt ? { prompt: right.prompt } : left.prompt ? { prompt: left.prompt } : {}),
|
|
398
|
+
...(right.meta !== undefined ? { meta: right.meta } : left.meta !== undefined ? { meta: left.meta } : {}),
|
|
399
|
+
...(right.usage ? { usage: right.usage } : left.usage ? { usage: left.usage } : {}),
|
|
400
|
+
visibility: right.visibility ?? left.visibility,
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
function runtimeEvaluationsCompatibleForExchange(left, right) {
|
|
404
|
+
if (runtimeRecordsEqual(left, right)) {
|
|
405
|
+
return true;
|
|
406
|
+
}
|
|
407
|
+
return runtimeRecordsEqual(runtimeEvaluationIdentityForExchange(left), runtimeEvaluationIdentityForExchange(right));
|
|
408
|
+
}
|
|
409
|
+
function runtimeEvaluationIdentityForExchange(evaluation) {
|
|
410
|
+
return {
|
|
411
|
+
id: evaluation.id,
|
|
412
|
+
runId: evaluation.runId,
|
|
413
|
+
candidateId: evaluation.candidateId,
|
|
414
|
+
candidateVersion: evaluation.candidateVersion,
|
|
415
|
+
benchmarkFingerprint: evaluation.benchmarkFingerprint,
|
|
416
|
+
candidateFingerprint: evaluation.candidateFingerprint,
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
function runtimeRunsCompatibleForExchange(left, right) {
|
|
420
|
+
if (runtimeRecordsEqual(left, right)) {
|
|
421
|
+
return true;
|
|
422
|
+
}
|
|
423
|
+
return runtimeRecordsEqual(runtimeRunIdentityForExchange(left), runtimeRunIdentityForExchange(right));
|
|
424
|
+
}
|
|
425
|
+
function runtimeRunIdentityForExchange(run) {
|
|
426
|
+
return {
|
|
427
|
+
id: run.id,
|
|
428
|
+
workflow: run.workflow,
|
|
429
|
+
benchmarkFingerprint: run.benchmarkFingerprint,
|
|
430
|
+
candidateId: run.candidateId ?? null,
|
|
431
|
+
outputCandidateId: run.outputCandidateId ?? null,
|
|
432
|
+
engineRun: run.engineRun,
|
|
433
|
+
improver: run.improver,
|
|
434
|
+
strategy: run.strategy,
|
|
435
|
+
budget: run.budget,
|
|
436
|
+
samples: run.samples,
|
|
437
|
+
attemptsRequested: run.attemptsRequested,
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
function runtimeJobIdentityForExchange(job) {
|
|
441
|
+
return {
|
|
442
|
+
id: job.id,
|
|
443
|
+
runId: job.runId,
|
|
444
|
+
candidateId: job.candidateId,
|
|
445
|
+
kind: job.kind,
|
|
446
|
+
attempt: job.attempt,
|
|
447
|
+
};
|
|
448
|
+
}
|
|
449
|
+
function compatibleRuntimeActiveCandidateId(candidates, candidateId, benchmarkFingerprint) {
|
|
450
|
+
if (!candidateId) {
|
|
451
|
+
return null;
|
|
452
|
+
}
|
|
453
|
+
const candidate = candidates.find((entry) => entry.id === candidateId) ?? null;
|
|
454
|
+
return candidate?.benchmarkFingerprint === benchmarkFingerprint ? candidate.id : null;
|
|
455
|
+
}
|
|
456
|
+
function latestCompatibleRuntimeCandidateId(candidates, benchmarkFingerprint) {
|
|
457
|
+
return candidates
|
|
458
|
+
.filter((candidate) => candidate.benchmarkFingerprint === benchmarkFingerprint &&
|
|
459
|
+
candidate.status === "evaluated")
|
|
460
|
+
.at(-1)?.id ?? null;
|
|
461
|
+
}
|
|
372
462
|
function canonicalRuntimeJson(value) {
|
|
373
463
|
if (Array.isArray(value)) {
|
|
374
464
|
return value.map(canonicalRuntimeJson);
|
|
@@ -431,6 +521,9 @@ function validateCandidateRecord(candidate) {
|
|
|
431
521
|
requireArchivePositiveInteger(candidate.ordinal, `candidate ${candidate.id}.ordinal`);
|
|
432
522
|
requireArchiveString(candidate.benchmarkFingerprint, `candidate ${candidate.id}.benchmarkFingerprint`);
|
|
433
523
|
requireArchiveString(candidate.candidateFingerprint, `candidate ${candidate.id}.candidateFingerprint`);
|
|
524
|
+
if (candidate.visibility !== "private" && candidate.visibility !== "public") {
|
|
525
|
+
throw new Error(`candidate ${candidate.id}.visibility must be private or public.`);
|
|
526
|
+
}
|
|
434
527
|
requireArchiveString(candidate.createdAt, `candidate ${candidate.id}.createdAt`);
|
|
435
528
|
}
|
|
436
529
|
function validateEvaluationRecord(evaluation) {
|
package/dist/project-source.d.ts
CHANGED
|
@@ -58,5 +58,6 @@ interface LocalProjectSourceOptions {
|
|
|
58
58
|
}
|
|
59
59
|
export declare function readLocalProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalProjectSource>;
|
|
60
60
|
export declare function readLocalAuthoredProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalAuthoredProjectSource>;
|
|
61
|
+
export declare function hostedEngineResolveFiles(source: LocalProjectSource): HostedFile[];
|
|
61
62
|
export {};
|
|
62
63
|
//# sourceMappingURL=project-source.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"project-source.d.ts","sourceRoot":"","sources":["../src/project-source.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,kCAAkC,EAGlC,KAAK,IAAI,EACT,KAAK,mBAAmB,EACzB,MAAM,8BAA8B,CAAC;AACtC,OAAO,
|
|
1
|
+
{"version":3,"file":"project-source.d.ts","sourceRoot":"","sources":["../src/project-source.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,kCAAkC,EAGlC,KAAK,IAAI,EACT,KAAK,mBAAmB,EACzB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAUL,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EAClC,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAGL,KAAK,qBAAqB,EAC3B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAML,KAAK,wBAAwB,EAC9B,MAAM,sBAAsB,CAAC;AAI9B,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAC5D,eAAO,MAAM,wBAAwB,eAAe,CAAC;AACrD,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC,OAAO,kCAAkC,CAAC,CAAC;IAC5D,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,UAAU,EAAE,CAAC;IAC9B,cAAc,EAAE,UAAU,EAAE,CAAC;IAC7B,kBAAkB,EAAE,UAAU,EAAE,CAAC;IACjC,QAAQ,EAAE,wBAAwB,EAAE,CAAC;IACrC,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,aAAa,EAAE,4BAA4B,CAAC;IAC5C,4BAA4B,EAAE,MAAM,CAAC;IACrC,wBAAwB,CAAC,EAAE,4BAA4B,CAAC,aAAa,CAAC,CAAC;IACvE,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,4BAA4B;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,CAAC,EAAE,IAAI,CAAC;CACb;AAED,UAAU,yBAAyB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAaD,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,kBAAkB,CAAC,CA8G7B;AAED,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,0BAA0B,CAAC,CA6BrC;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,kBAAkB,GAAG,UAAU,EAAE,CAqBjF"}
|