@workbench-ai/workbench 0.0.51 → 0.0.52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA2IA,UAAU,KAAK;IACb,KAAK,EAAE,MAAM,CAAC,cAAc,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;CAC/B;AA4BD,UAAU,iBAAiB;CAAG;AAuK9B,wBAAsB,MAAM,CAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,GAAE,KAIH,EACD,cAAc,GAAE,iBAAsB,GACrC,OAAO,CAAC,MAAM,CAAC,CAmHjB"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AA0IA,UAAU,KAAK;IACb,KAAK,EAAE,MAAM,CAAC,cAAc,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;CAC/B;AA4BD,UAAU,iBAAiB;CAAG;AA4K9B,wBAAsB,MAAM,CAC1B,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,EAAE,GAAE,KAIH,EACD,cAAc,GAAE,iBAAsB,GACrC,OAAO,CAAC,MAAM,CAAC,CAmHjB"}
package/dist/index.js CHANGED
@@ -6,7 +6,7 @@ import os from "node:os";
6
6
  import path from "node:path";
7
7
  import { Writable } from "node:stream";
8
8
  import { createCandidateFilePreview, createBaselineCandidateJob as createRuntimeBaselineCandidateJob, evaluationScorecardId, evaluationMeanMetrics, executeWorkbenchExecutionJob, engineResolveBindingForSpec, filterOptimizerTraceJobsForCaseIds, filterCandidateSourceFiles, formatWorkbenchCaseSelector, formatWorkbenchSelectionPolicy, workbenchCaseSelectorUsesAllCases, workbenchExecutionPurpose, workbenchRunExecutionFingerprint, createWorkbenchAdapterAuthBundle, createOptimizerTraceInputFiles, DOCKER_SANDBOX_BACKEND, localWorkbenchAdapterAuthStore, materializeWorkbenchRunResult, normalizeSurfaceFiles, planWorkbenchExecutionJobsForPurpose, runWorkbenchExecutionDag, resolveEngineCaseExecutionConfig, resolveWorkbenchResolvedSourceYaml, summarizeCandidateFiles, validateWorkbenchRunEnvelope, parseWorkbenchAdapterAuthTarget, workbenchEngineCaseIdsForImproveEvaluation, workbenchEngineCaseIdsForSelector, workbenchImproveOptimizeSelector, workbenchImproveSelectionPolicy, workbenchProjectSourceFingerprint, workbenchRuntimeBundleFingerprint, } from "@workbench-ai/workbench-core";
9
- import { assertWorkbenchAdapterOperationResultOk, collectWorkbenchAdapterAuthRequirements, WORKBENCH_ADAPTER_RESULT_FILE, WORKBENCH_ADAPTER_RESULT_PROTOCOL, normalizeWorkbenchAdapterOperationRequest, readWorkbenchAdapterOperationResult, workbenchAdapterOperationCommand, workbenchAdapterOperationResultPath, withDefaultWorkbenchAdapterAuthProfiles as applyDefaultWorkbenchAdapterAuthProfiles, } from "@workbench-ai/workbench-protocol";
9
+ import { assertWorkbenchAdapterOperationResultOk, collectWorkbenchAdapterAuthRequirements, normalizeWorkbenchAdapterOperationRequest, readWorkbenchAdapterOperationResult, workbenchAdapterOperationCommand, workbenchAdapterOperationResultPath, withDefaultWorkbenchAdapterAuthProfiles as applyDefaultWorkbenchAdapterAuthProfiles, } from "@workbench-ai/workbench-protocol";
10
10
  import { builtinLocalTraceAdapter, builtinLocalTraceAdapters, sortLocalTraceRefs, } from "@workbench-ai/workbench-built-in-adapters/local-traces";
11
11
  import { commandUsage, HOSTED_WATCH_LIFECYCLE_NOTE, LOCAL_DEV_OPEN_LIFECYCLE_NOTE, rootUsage, } from "./command-model.js";
12
12
  import { startLocalWorkbenchDevServer } from "./dev-open-server.js";
@@ -15,7 +15,7 @@ import { defaultAdapterManifests, composeRuntimeDockerfileWithAdapters, resolveD
15
15
  import { createAdapterCommandEnv } from "./adapter-command-env.js";
16
16
  import { loadLocalArchive, loadLocalArchiveIndex, exportLocalRuntimeBundle, importLocalRuntimeBundle, runtimeBundleStats, materializeCandidateRoot, readLocalCandidate, readLocalCandidateFiles, readLocalJobs, saveLocalArchive, saveLocalJobs, setLocalActive, upsertLocalRun, upsertLocalCandidate, upsertLocalEvaluation, } from "./local-archive.js";
17
17
  import { WorkspaceSnapshotError, } from "./workspace-snapshot.js";
18
- import { readLocalProjectSource, WORKBENCH_BENCHMARK_FILE, } from "./project-source.js";
18
+ import { hostedEngineResolveFiles, readLocalProjectSource, WORKBENCH_BENCHMARK_FILE, } from "./project-source.js";
19
19
  import { localBenchmarkFingerprint, localCandidateFingerprint, } from "./benchmark-fingerprint.js";
20
20
  const require = createRequire(import.meta.url);
21
21
  function getCliVersion() {
@@ -869,14 +869,15 @@ async function localRun(argv, io, runtimeOptions) {
869
869
  },
870
870
  });
871
871
  for (const candidate of materialized.candidates) {
872
- outputCandidateId = candidate.id;
873
- snapshot = upsertLocalCandidate(snapshot, candidate, materialized.candidateFiles[candidate.id] ?? []);
874
- events.push(createLocalEvent("candidate_created", candidate.createdAt, {
872
+ const localCandidate = localCandidateRecord(candidate);
873
+ outputCandidateId = localCandidate.id;
874
+ snapshot = upsertLocalCandidate(snapshot, localCandidate, materialized.candidateFiles[localCandidate.id] ?? []);
875
+ events.push(createLocalEvent("candidate_created", localCandidate.createdAt, {
875
876
  runId,
876
- candidateId: candidate.id,
877
- baseId: candidate.baseId,
878
- status: candidate.status,
879
- metrics: evaluationMeanMetrics(candidate.eval),
877
+ candidateId: localCandidate.id,
878
+ baseId: localCandidate.baseId,
879
+ status: localCandidate.status,
880
+ metrics: evaluationMeanMetrics(localCandidate.eval),
880
881
  }));
881
882
  }
882
883
  for (const evaluation of materialized.evaluations) {
@@ -1280,7 +1281,7 @@ async function localEvaluateCandidate(argv, io, runtimeOptions) {
1280
1281
  previousCandidate: existingCandidate ?? null,
1281
1282
  existingCandidateCount: snapshot.candidates.length,
1282
1283
  });
1283
- for (const candidateRecord of materialized.candidates) {
1284
+ for (const candidateRecord of materialized.candidates.map(localCandidateRecord)) {
1284
1285
  snapshot = upsertLocalCandidate(snapshot, candidateRecord, materialized.candidateFiles[candidateRecord.id] ?? []);
1285
1286
  }
1286
1287
  if (materialized.activeCandidateId) {
@@ -3005,8 +3006,6 @@ async function pushBenchmark(argv, io) {
3005
3006
  const nextOrigin = await writeWorkbenchOriginFromState(dir, {
3006
3007
  baseUrl,
3007
3008
  state: response.state,
3008
- project: publishedProject,
3009
- sourceFingerprint: state.source.fingerprint,
3010
3009
  });
3011
3010
  writeOutput({
3012
3011
  ok: true,
@@ -3040,8 +3039,6 @@ async function createHostedBenchmarkFromState(args) {
3040
3039
  const origin = await writeWorkbenchOriginFromState(args.dir, {
3041
3040
  baseUrl: args.baseUrl,
3042
3041
  state: result.state,
3043
- project,
3044
- sourceFingerprint: args.state.source.fingerprint,
3045
3042
  });
3046
3043
  return { project, origin, result };
3047
3044
  }
@@ -3150,11 +3147,11 @@ async function applyProjectStateToLocal(args) {
3150
3147
  await assertLocalSourceMatchesOrigin(args.dir, args.origin);
3151
3148
  }
3152
3149
  await syncSourceFiles(args.dir, args.state.source.files);
3153
- const runtimeImport = await importLocalRuntimeBundle(args.dir, args.state.runtime);
3150
+ const benchmarkFingerprint = localBenchmarkFingerprint(await readLocalProjectSource(args.dir));
3151
+ const runtimeImport = await importLocalRuntimeBundle(args.dir, args.state.runtime, benchmarkFingerprint);
3154
3152
  const origin = await writeWorkbenchOriginFromState(args.dir, {
3155
3153
  baseUrl: args.baseUrl,
3156
3154
  state: args.state,
3157
- sourceFingerprint: await localSourceFingerprint(args.dir),
3158
3155
  });
3159
3156
  return {
3160
3157
  origin,
@@ -3193,20 +3190,21 @@ async function retryHostedWorkflow(argv, io) {
3193
3190
  method: "POST",
3194
3191
  body: retryTarget.request,
3195
3192
  }, target.baseUrl);
3196
- const startedRun = withRunUrls(target, response.run);
3193
+ const runTarget = hostedTargetForRunStartResponse(target, response);
3194
+ const startedRun = withRunUrls(runTarget, response.run);
3197
3195
  if (parsed.flags.watch === true) {
3198
3196
  if (parsed.flags.json !== true) {
3199
3197
  io.stdout.write(`${formatHostedRunStarted(startedRun, retryTarget.workflow).trimEnd()}\n${HOSTED_WATCH_LIFECYCLE_NOTE}\n`);
3200
3198
  }
3201
3199
  const watched = await watchHostedRun({
3202
3200
  parsed,
3203
- target,
3201
+ target: runTarget,
3204
3202
  runId: response.run.id,
3205
3203
  intervalMs: watchIntervalMs ?? 1000,
3206
3204
  timeoutMs: watchTimeoutMs,
3207
3205
  });
3208
- const outputRun = withRunUrls(target, await withHostedRunFailureSummary(target, watched));
3209
- await tryImportTerminalHostedProjectState({ target, io });
3206
+ const outputRun = withRunUrls(runTarget, await withHostedRunFailureSummary(runTarget, watched));
3207
+ await tryImportTerminalHostedProjectState({ target: runTarget, io });
3210
3208
  const result = {
3211
3209
  ok: hostedRunSucceeded(watched),
3212
3210
  retried: {
@@ -3377,12 +3375,10 @@ function hostedRunRecordFailed(run) {
3377
3375
  }
3378
3376
  async function startHostedWorkflow(workflow, argv, io) {
3379
3377
  const parsed = parseArgs(argv);
3380
- rejectUnknownFlags(parsed, new Set([
3378
+ const allowedFlags = new Set([
3381
3379
  "dir",
3382
3380
  "benchmark",
3383
- "base",
3384
3381
  "runs",
3385
- "budget",
3386
3382
  "samples",
3387
3383
  "rerun",
3388
3384
  "watch",
@@ -3390,7 +3386,15 @@ async function startHostedWorkflow(workflow, argv, io) {
3390
3386
  "interval-ms",
3391
3387
  "timeout-ms",
3392
3388
  "json",
3393
- ]));
3389
+ ]);
3390
+ if (workflow === "eval") {
3391
+ allowedFlags.add("candidate");
3392
+ }
3393
+ else {
3394
+ allowedFlags.add("base");
3395
+ allowedFlags.add("budget");
3396
+ }
3397
+ rejectUnknownFlags(parsed, allowedFlags);
3394
3398
  if (parsed.positionals.length > 1) {
3395
3399
  throw new UsageError(`workbench ${workflow} --hosted accepts at most one source file or directory argument.`);
3396
3400
  }
@@ -3431,25 +3435,27 @@ async function startHostedWorkflow(workflow, argv, io) {
3431
3435
  }, parsed, io, () => `Processed ${selectedRunIds.length} hosted candidate run(s); ${failed} failed.`);
3432
3436
  return failed === 0 ? 0 : 1;
3433
3437
  }
3434
- const baseCandidateId = asOptionalString(parsed.flags.base);
3438
+ const selectedCandidateId = workflow === "eval"
3439
+ ? asOptionalString(parsed.flags.candidate)
3440
+ : asOptionalString(parsed.flags.base);
3435
3441
  const request = workflow === "improve"
3436
3442
  ? {
3437
3443
  workflow,
3438
3444
  budget,
3439
3445
  samples,
3440
- ...(baseCandidateId ? { candidateId: baseCandidateId } : {}),
3446
+ ...(selectedCandidateId ? { candidateId: selectedCandidateId } : {}),
3441
3447
  }
3442
3448
  : {
3443
3449
  workflow,
3444
3450
  samples,
3445
- ...(baseCandidateId ? { candidateId: baseCandidateId } : {}),
3451
+ ...(selectedCandidateId ? { candidateId: selectedCandidateId } : {}),
3446
3452
  };
3447
3453
  const projectSource = selectedRunIds[0] === defaultProjectSource.candidateRunId
3448
3454
  ? defaultProjectSource
3449
3455
  : await readLocalProjectSource(path.resolve(sourceArg), { runId: selectedRunIds[0] });
3450
3456
  request.sourceYaml = projectSource.specSource;
3451
3457
  request.adapterFiles = projectSource.adapterFiles;
3452
- if (workflow === "eval" && !baseCandidateId) {
3458
+ if (workflow === "eval" && !selectedCandidateId) {
3453
3459
  request.candidateFiles = projectSource.candidateFiles;
3454
3460
  }
3455
3461
  if (parsed.flags.rerun === true) {
@@ -3484,7 +3490,7 @@ async function startHostedWorkflow(workflow, argv, io) {
3484
3490
  parsed,
3485
3491
  target,
3486
3492
  samples: request.samples,
3487
- candidateId: baseCandidateId,
3493
+ candidateId: selectedCandidateId,
3488
3494
  sourceYaml: projectSource.specSource,
3489
3495
  adapterFiles: projectSource.adapterFiles,
3490
3496
  intervalMs: watchIntervalMs ?? 1000,
@@ -3496,12 +3502,13 @@ async function startHostedWorkflow(workflow, argv, io) {
3496
3502
  method: "POST",
3497
3503
  body: request,
3498
3504
  }, target.baseUrl);
3499
- const startedRun = withRunUrls(target, response.run);
3505
+ const runTarget = hostedTargetForRunStartResponse(target, response);
3506
+ const startedRun = withRunUrls(runTarget, response.run);
3500
3507
  const startedRunOutput = response.reused === true
3501
3508
  ? { ...startedRun, reused: true }
3502
3509
  : startedRun;
3503
3510
  if (response.reused === true && response.run.status === "finished") {
3504
- await tryImportTerminalHostedProjectState({ target, io });
3511
+ await tryImportTerminalHostedProjectState({ target: runTarget, io });
3505
3512
  writeOutput({
3506
3513
  ok: hostedRunSucceeded(response.run),
3507
3514
  reused: true,
@@ -3517,14 +3524,14 @@ async function startHostedWorkflow(workflow, argv, io) {
3517
3524
  }
3518
3525
  const watched = await watchHostedRun({
3519
3526
  parsed,
3520
- target,
3527
+ target: runTarget,
3521
3528
  runId: response.run.id,
3522
3529
  intervalMs: watchIntervalMs ?? 1000,
3523
3530
  timeoutMs: watchTimeoutMs,
3524
3531
  });
3525
- const outputRun = await withHostedRunFailureSummary(target, watched);
3526
- await tryImportTerminalHostedProjectState({ target, io });
3527
- writeOutput(withRunUrls(target, outputRun), parsed, io, formatHostedRunResult);
3532
+ const outputRun = await withHostedRunFailureSummary(runTarget, watched);
3533
+ await tryImportTerminalHostedProjectState({ target: runTarget, io });
3534
+ writeOutput(withRunUrls(runTarget, outputRun), parsed, io, formatHostedRunResult);
3528
3535
  return hostedRunSucceeded(watched) ? 0 : 1;
3529
3536
  }
3530
3537
  writeOutput(startedRunOutput, parsed, io, (run) => formatHostedRunStarted(run, workflow).trimEnd());
@@ -3556,9 +3563,10 @@ async function ensureHostedImproveBaseCandidate(args) {
3556
3563
  ...(args.adapterFiles.length > 0 ? { adapterFiles: args.adapterFiles } : {}),
3557
3564
  },
3558
3565
  }, args.target.baseUrl);
3566
+ const runTarget = hostedTargetForRunStartResponse(args.target, response);
3559
3567
  const watched = await watchHostedRun({
3560
3568
  parsed: args.parsed,
3561
- target: args.target,
3569
+ target: runTarget,
3562
3570
  runId: response.run.id,
3563
3571
  intervalMs: args.intervalMs,
3564
3572
  timeoutMs: args.timeoutMs,
@@ -3569,15 +3577,14 @@ async function ensureHostedImproveBaseCandidate(args) {
3569
3577
  if (!watched.candidateId) {
3570
3578
  throw new UsageError(`Parent candidate eval ${watched.id} did not produce a candidate.`);
3571
3579
  }
3572
- await tryImportTerminalHostedProjectState({ target: args.target, io: args.io });
3580
+ await tryImportTerminalHostedProjectState({ target: runTarget, io: args.io });
3573
3581
  return watched.candidateId;
3574
3582
  }
3575
3583
  function hostedWorkflowArgsForRun(args) {
3576
3584
  const next = ["--dir", args.sourceDir, "--runs", args.runId, "--json"];
3577
3585
  appendStringFlag(next, "benchmark", asOptionalString(args.parsed.flags.benchmark));
3578
- appendStringFlag(next, "base", asOptionalString(args.parsed.flags.base));
3586
+ appendStringFlag(next, "candidate", asOptionalString(args.parsed.flags.candidate));
3579
3587
  appendStringFlag(next, "samples", asOptionalString(args.parsed.flags.samples));
3580
- appendStringFlag(next, "budget", asOptionalString(args.parsed.flags.budget));
3581
3588
  appendStringFlag(next, "interval-ms", asOptionalString(args.parsed.flags["interval-ms"]));
3582
3589
  appendStringFlag(next, "timeout-ms", asOptionalString(args.parsed.flags["timeout-ms"]));
3583
3590
  if (args.parsed.flags.watch === true) {
@@ -3814,6 +3821,9 @@ async function resolveRemoteProject(projectRef, baseUrl) {
3814
3821
  return response.benchmark;
3815
3822
  }
3816
3823
  function withRunUrls(target, run) {
3824
+ if (!target.owner || !target.projectName) {
3825
+ return { ...run };
3826
+ }
3817
3827
  return {
3818
3828
  ...run,
3819
3829
  urls: buildWorkbenchResourceUrls(target, {
@@ -3822,6 +3832,31 @@ function withRunUrls(target, run) {
3822
3832
  }),
3823
3833
  };
3824
3834
  }
3835
+ function hostedTargetForRunStartResponse(target, response) {
3836
+ const projectId = response.benchmark?.id ?? response.run.projectId ?? target.projectId;
3837
+ if (projectId === target.projectId && !response.benchmark) {
3838
+ return target;
3839
+ }
3840
+ const origin = target.origin?.projectId === projectId ? target.origin : null;
3841
+ const next = {
3842
+ ...target,
3843
+ projectId,
3844
+ origin,
3845
+ };
3846
+ if (response.benchmark?.ownerUsername) {
3847
+ next.owner = response.benchmark.ownerUsername;
3848
+ }
3849
+ else {
3850
+ delete next.owner;
3851
+ }
3852
+ if (response.benchmark?.name) {
3853
+ next.projectName = response.benchmark.name;
3854
+ }
3855
+ else {
3856
+ delete next.projectName;
3857
+ }
3858
+ return next;
3859
+ }
3825
3860
  function hostedRunEvaluationCandidateId(run, jobs = []) {
3826
3861
  if (run.outputCandidateId) {
3827
3862
  return run.outputCandidateId;
@@ -3834,7 +3869,8 @@ function hostedRunEvaluationCandidateId(run, jobs = []) {
3834
3869
  }
3835
3870
  function localProjectState(args) {
3836
3871
  const stateSource = localProjectStateSource(args.source);
3837
- const runtimeFingerprint = workbenchRuntimeBundleFingerprint(args.runtime);
3872
+ const runtime = runtimeBundleForProjectVisibility(args.runtime, args.visibility);
3873
+ const runtimeFingerprint = workbenchRuntimeBundleFingerprint(runtime);
3838
3874
  return {
3839
3875
  schema: "workbench.project.state.v1",
3840
3876
  project: {
@@ -3850,7 +3886,22 @@ function localProjectState(args) {
3850
3886
  runtimeFingerprint: args.origin?.runtimeFingerprint ?? runtimeFingerprint,
3851
3887
  },
3852
3888
  source: stateSource,
3853
- runtime: args.runtime,
3889
+ runtime,
3890
+ };
3891
+ }
3892
+ function localCandidateRecord(candidate) {
3893
+ return {
3894
+ ...candidate,
3895
+ visibility: "private",
3896
+ };
3897
+ }
3898
+ function runtimeBundleForProjectVisibility(runtime, visibility) {
3899
+ return {
3900
+ ...runtime,
3901
+ candidates: runtime.candidates.map((candidate) => ({
3902
+ ...candidate,
3903
+ visibility,
3904
+ })),
3854
3905
  };
3855
3906
  }
3856
3907
  function localProjectStateSource(source) {
@@ -3888,8 +3939,6 @@ function hostedProjectSummaryFromState(state) {
3888
3939
  ownerUsername: state.project.ownerUsername,
3889
3940
  name: state.project.name,
3890
3941
  visibility: state.project.visibility,
3891
- currentSpecVersionId: state.source.revisionId ?? state.base.sourceRevisionId,
3892
- sourceFingerprint: state.source.fingerprint ?? state.base.sourceFingerprint,
3893
3942
  };
3894
3943
  }
3895
3944
  function sourceFileCount(source) {
@@ -3910,28 +3959,6 @@ function hostedProjectSourceRequest(source) {
3910
3959
  resources,
3911
3960
  };
3912
3961
  }
3913
- function hostedEngineResolveFiles(source) {
3914
- return [
3915
- ...source.engineResolveFiles,
3916
- {
3917
- path: WORKBENCH_ADAPTER_RESULT_FILE,
3918
- content: `${JSON.stringify({
3919
- protocol: WORKBENCH_ADAPTER_RESULT_PROTOCOL,
3920
- operation: "engine.resolve",
3921
- ok: true,
3922
- value: {
3923
- cases: source.engineCases,
3924
- ...(source.engineResolveEnvironment
3925
- ? { environment: source.engineResolveEnvironment }
3926
- : {}),
3927
- },
3928
- feedback: {
3929
- path: source.engineResolveFingerprintPath,
3930
- },
3931
- }, null, 2)}\n`,
3932
- },
3933
- ];
3934
- }
3935
3962
  function isRemoteProjectId(value) {
3936
3963
  return /^wb_[a-f0-9]{12}$/u.test(value);
3937
3964
  }
@@ -4147,14 +4174,11 @@ async function writeWorkbenchOrigin(dir, input) {
4147
4174
  return origin;
4148
4175
  }
4149
4176
  async function writeWorkbenchOriginFromState(dir, args) {
4150
- const owner = args.project?.ownerUsername ?? args.state.project.ownerUsername;
4151
- const name = args.project?.name ?? args.state.project.name;
4152
- const sourceRevisionId = args.project?.currentSpecVersionId ??
4153
- args.state.source.revisionId ??
4177
+ const owner = args.state.project.ownerUsername;
4178
+ const name = args.state.project.name;
4179
+ const sourceRevisionId = args.state.source.revisionId ??
4154
4180
  args.state.base.sourceRevisionId;
4155
- const sourceFingerprint = args.sourceFingerprint ??
4156
- args.project?.sourceFingerprint ??
4157
- args.state.source.fingerprint ??
4181
+ const sourceFingerprint = args.state.source.fingerprint ??
4158
4182
  args.state.base.sourceFingerprint;
4159
4183
  const runtimeFingerprint = args.state.base.runtimeFingerprint ??
4160
4184
  workbenchRuntimeBundleFingerprint(args.state.runtime);
@@ -4164,16 +4188,12 @@ async function writeWorkbenchOriginFromState(dir, args) {
4164
4188
  return await writeWorkbenchOrigin(dir, {
4165
4189
  baseUrl: args.baseUrl,
4166
4190
  remote: `${owner}/${name}`,
4167
- projectId: args.project?.id ?? args.state.project.id,
4191
+ projectId: args.state.project.id,
4168
4192
  sourceRevisionId,
4169
4193
  sourceFingerprint,
4170
4194
  runtimeFingerprint,
4171
4195
  });
4172
4196
  }
4173
- async function localSourceFingerprint(dir) {
4174
- const source = localProjectStateSource(await readLocalProjectSource(dir));
4175
- return source.fingerprint ?? workbenchProjectSourceFingerprint(source);
4176
- }
4177
4197
  function parseOriginRemote(origin) {
4178
4198
  return parseRemoteName(origin.remote);
4179
4199
  }
@@ -24,7 +24,7 @@ export declare function loadLocalArchiveIndex(workspace: string): Promise<LocalA
24
24
  export declare function saveLocalArchive(workspace: string, snapshot: LocalArchiveSnapshot): Promise<void>;
25
25
  export declare function saveLocalJobs(workspace: string, jobs: readonly HostedWorkbenchJob[]): Promise<void>;
26
26
  export declare function exportLocalRuntimeBundle(workspace: string): Promise<WorkbenchRuntimeBundle>;
27
- export declare function importLocalRuntimeBundle(workspace: string, bundle: WorkbenchRuntimeBundle): Promise<WorkbenchRuntimeImportResult>;
27
+ export declare function importLocalRuntimeBundle(workspace: string, bundle: WorkbenchRuntimeBundle, currentBenchmarkFingerprint: string): Promise<WorkbenchRuntimeImportResult>;
28
28
  export declare function runtimeBundleStats(bundle: WorkbenchRuntimeBundle): WorkbenchRuntimeBundleStats;
29
29
  export declare function sanitizeRuntimeJobForExchange(job: HostedWorkbenchJob): HostedWorkbenchJob;
30
30
  export declare function readLocalExecutionFiles(workspace: string, jobId: string): Promise<SurfaceSnapshotFile[]>;
@@ -1 +1 @@
1
- {"version":3,"file":"local-archive.d.ts","sourceRoot":"","sources":["../src/local-archive.ts"],"names":[],"mappings":"AAGA,OAAO,EAQL,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,EAC3B,MAAM,8BAA8B,CAAC;AAOtC,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;IACtD,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG;IAClD,KAAK,CAAC,EAAE,uBAAuB,CAAC;IAChC,aAAa,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACzC,CAAC;AASF,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAevF;AAED,wBAAsB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkBzF;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,kBAAkB,EAAE,GAClC,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,sBAAsB,CAAC,CAuBjC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAAC,4BAA4B,CAAC,CA2GvC;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,sBAAsB,GAC7B,2BAA2B,CAE7B;AAED,wBAAgB,6BAA6B,CAC3C,GAAG,EAAE,kBAAkB,GACtB,kBAAkB,CAEpB;AAyDD,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAIhC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,CAAC,CAU1B;AAED,wBAAsB,4BAA4B,CAChD,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAKhC;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,mBAAmB,CAAC,CAU9B;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,UAAU,CAAC,CAUrB;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAM7B;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAE7B;AAED,wBAAsB,iBAAiB,CACrC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAElC;AAED,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,oBAAoB,EAC9B,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,oBAAoB,CAYtB;AAED,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,oBAAoB,EAC9B,UAAU,EAAE,mBAAmB,GAC9B,oBAAoB,CAQtB;AAED,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,oBAAoB,EAC9B,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,SAAS,YAAY,EAAE,GAC9B,oBAAoB,CAYtB;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,oBAAoB,CAK5G;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,GAAG,eAAe,CAMvG;AAED,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,GAAG,mBAAmB,EAAE,CAGlH;AA+eD,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,OAAO,CAAC,MAAM,EAAE,CAAC,CAOnB;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,mBAAmB,EAAE,EACrC,QAAQ,EAAE,MAAM,GACf,mBAAmB,GAAG,IAAI,CAG5B"}
1
+ {"version":3,"file":"local-archive.d.ts","sourceRoot":"","sources":["../src/local-archive.ts"],"names":[],"mappings":"AAGA,OAAO,EAQL,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,qBAAqB,EAC3B,MAAM,8BAA8B,CAAC;AAOtC,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;IACtD,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GAAG,kBAAkB,GAAG;IAClD,KAAK,CAAC,EAAE,uBAAuB,CAAC;IAChC,aAAa,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACzC,CAAC;AASF,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAevF;AAED,wBAAsB,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAkBzF;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,kBAAkB,EAAE,GAClC,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,sBAAsB,CAAC,CAuBjC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,sBAAsB,EAC9B,2BAA2B,EAAE,MAAM,GAClC,OAAO,CAAC,4BAA4B,CAAC,CAkHvC;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,sBAAsB,GAC7B,2BAA2B,CAE7B;AAED,wBAAgB,6BAA6B,CAC3C,GAAG,EAAE,kBAAkB,GACtB,kBAAkB,CAEpB;AAyDD,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAIhC;AAED,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,CAAC,CAU1B;AAED,wBAAsB,4BAA4B,CAChD,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAKhC;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,mBAAmB,CAAC,CAU9B;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,UAAU,CAAC,CAUrB;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAM7B;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAE7B;AAED,wBAAsB,iBAAiB,CACrC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAElC;AAED,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,oBAAoB,EAC9B,SAAS,EAAE,eAAe,EAC1B,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,oBAAoB,CAYtB;AAED,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,oBAAoB,EAC9B,UAAU,EAAE,mBAAmB,GAC9B,oBAAoB,CAQtB;AAED,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,oBAAoB,EAC9B,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,SAAS,YAAY,EAAE,GAC9B,oBAAoB,CAYtB;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,oBAAoB,CAK5G;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,GAAG,eAAe,CAMvG;AAED,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,GAAG,mBAAmB,EAAE,CAGlH;AAkoBD,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,OAAO,CAAC,MAAM,EAAE,CAAC,CAOnB;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,mBAAmB,EAAE,EACrC,QAAQ,EAAE,MAAM,GACf,mBAAmB,GAAG,IAAI,CAG5B"}
@@ -93,7 +93,7 @@ export async function exportLocalRuntimeBundle(workspace) {
93
93
  events: snapshot.events.map((event) => ({ ...event })),
94
94
  };
95
95
  }
96
- export async function importLocalRuntimeBundle(workspace, bundle) {
96
+ export async function importLocalRuntimeBundle(workspace, bundle, currentBenchmarkFingerprint) {
97
97
  validateRuntimeBundleSchema(bundle);
98
98
  const snapshot = await loadLocalArchive(workspace);
99
99
  const existingJobs = (await readLocalJobs(workspace)).map(sanitizeRuntimeJobForExchange);
@@ -105,7 +105,7 @@ export async function importLocalRuntimeBundle(workspace, bundle) {
105
105
  const incomingCandidates = bundle.candidates.map(sanitizeWorkbenchRuntimeCandidateForExchange);
106
106
  const candidates = mergeRecordsById(existingCandidates, incomingCandidates, (candidate) => candidate.id, (didChange) => {
107
107
  changed ||= didChange;
108
- }).sort(compareLocalCandidateRecords);
108
+ }, runtimeCandidatesCompatibleForExchange, mergeRuntimeCandidateForExchange).sort(compareLocalCandidateRecords);
109
109
  const candidateFiles = { ...snapshot.candidateFiles };
110
110
  for (const group of bundle.candidateFiles) {
111
111
  const candidateId = localRecordName(group.candidateId);
@@ -121,21 +121,18 @@ export async function importLocalRuntimeBundle(workspace, bundle) {
121
121
  }
122
122
  candidateFiles[candidateId] = files;
123
123
  }
124
- const candidateIds = new Set(candidates.map((candidate) => candidate.id));
125
- const activeId = bundle.activeId && candidateIds.has(bundle.activeId)
126
- ? bundle.activeId
127
- : snapshot.activeId && candidateIds.has(snapshot.activeId)
128
- ? snapshot.activeId
129
- : null;
124
+ const activeId = compatibleRuntimeActiveCandidateId(candidates, bundle.activeId ?? null, currentBenchmarkFingerprint) ??
125
+ compatibleRuntimeActiveCandidateId(candidates, snapshot.activeId, currentBenchmarkFingerprint) ??
126
+ latestCompatibleRuntimeCandidateId(candidates, currentBenchmarkFingerprint);
130
127
  if (activeId !== snapshot.activeId) {
131
128
  changed = true;
132
129
  }
133
130
  const evaluations = mergeRecordsById(snapshot.evaluations, bundle.evaluations, (evaluation) => evaluation.id, (didChange) => {
134
131
  changed ||= didChange;
135
- }).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
132
+ }, runtimeEvaluationsCompatibleForExchange).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id));
136
133
  const runs = mergeRecordsById(snapshot.runs, bundle.runs, (run) => run.id, (didChange) => {
137
134
  changed ||= didChange;
138
- }).sort((left, right) => left.startedAt.localeCompare(right.startedAt) || left.id.localeCompare(right.id));
135
+ }, runtimeRunsCompatibleForExchange).sort((left, right) => left.startedAt.localeCompare(right.startedAt) || left.id.localeCompare(right.id));
139
136
  const events = mergeRecordsById(snapshot.events, bundle.events, runtimeEventKey, (didChange) => {
140
137
  changed ||= didChange;
141
138
  }).sort((left, right) => left.at.localeCompare(right.at) || left.id.localeCompare(right.id));
@@ -143,13 +140,20 @@ export async function importLocalRuntimeBundle(workspace, bundle) {
143
140
  await Promise.all(existingJobs.map(async (job) => {
144
141
  executionFilesByJobId.set(job.id, await readLocalExecutionFiles(workspace, job.id));
145
142
  }));
143
+ const existingJobById = new Map(existingJobs.map((job) => [job.id, job]));
144
+ const incomingJobById = new Map(bundle.jobs.map(sanitizeRuntimeJobForExchange).map((job) => [job.id, job]));
146
145
  for (const group of bundle.executionFiles) {
147
146
  const jobId = localRecordName(group.jobId);
148
147
  const files = copySurfaceFiles(group.files);
149
148
  const existing = executionFilesByJobId.get(jobId);
150
149
  if (existing) {
151
150
  if (!workbenchSurfaceFilesEqualForExchange(existing, files)) {
152
- throw new Error(`Runtime history conflict for execution files ${jobId}.`);
151
+ const existingJob = existingJobById.get(jobId) ?? null;
152
+ const incomingJob = incomingJobById.get(jobId) ?? null;
153
+ if (!existingJob || !incomingJob || !runtimeJobsEqualForExchange(existingJob, incomingJob)) {
154
+ throw new Error(`Runtime history conflict for execution files ${jobId}.`);
155
+ }
156
+ changed = true;
153
157
  }
154
158
  }
155
159
  else {
@@ -332,7 +336,7 @@ function validateRuntimeBundleSchema(bundle) {
332
336
  throw new Error("Unsupported Workbench runtime bundle.");
333
337
  }
334
338
  }
335
- function mergeRecordsById(existing, incoming, idFor, markChanged, equal = runtimeRecordsEqual) {
339
+ function mergeRecordsById(existing, incoming, idFor, markChanged, equal = runtimeRecordsEqual, merge = (_left, right) => right) {
336
340
  const records = new Map();
337
341
  for (const record of existing) {
338
342
  records.set(localRecordName(idFor(record)), record);
@@ -340,13 +344,19 @@ function mergeRecordsById(existing, incoming, idFor, markChanged, equal = runtim
340
344
  for (const record of incoming) {
341
345
  const id = localRecordName(idFor(record));
342
346
  const previous = records.get(id);
343
- if (!previous || !equal(previous, record)) {
344
- if (previous) {
345
- throw new Error(`Runtime history conflict for id ${id}.`);
346
- }
347
+ if (!previous) {
348
+ markChanged(true);
349
+ records.set(id, record);
350
+ continue;
351
+ }
352
+ if (!equal(previous, record)) {
353
+ throw new Error(`Runtime history conflict for id ${id}.`);
354
+ }
355
+ const merged = merge(previous, record);
356
+ if (!runtimeRecordsEqual(previous, merged)) {
347
357
  markChanged(true);
348
358
  }
349
- records.set(id, record);
359
+ records.set(id, merged);
350
360
  }
351
361
  return [...records.values()];
352
362
  }
@@ -355,7 +365,10 @@ function runtimeRecordsEqual(left, right) {
355
365
  JSON.stringify(canonicalRuntimeJson(right));
356
366
  }
357
367
  function runtimeJobsEqualForExchange(left, right) {
358
- return runtimeRecordsEqual(runtimeComparableJob(left), runtimeComparableJob(right));
368
+ if (runtimeRecordsEqual(runtimeComparableJob(left), runtimeComparableJob(right))) {
369
+ return true;
370
+ }
371
+ return runtimeRecordsEqual(runtimeJobIdentityForExchange(left), runtimeJobIdentityForExchange(right));
359
372
  }
360
373
  function runtimeComparableJob(job) {
361
374
  const comparable = sanitizeRuntimeJobForExchange(job);
@@ -369,6 +382,83 @@ function runtimeComparableJob(job) {
369
382
  output: portableOutput,
370
383
  };
371
384
  }
385
+ function runtimeCandidatesCompatibleForExchange(left, right) {
386
+ return runtimeRecordsEqual(runtimeCandidateIdentityForExchange(left), runtimeCandidateIdentityForExchange(right));
387
+ }
388
+ function runtimeCandidateIdentityForExchange(candidate) {
389
+ const { eval: _eval, prompt: _prompt, meta: _meta, status: _status, usage: _usage, visibility: _visibility, ownerUserId: _ownerUserId, ownerUsername: _ownerUsername, metrics: _metrics, candidateRunId: _candidateRunId, candidateRunName: _candidateRunName, ...identity } = candidate;
390
+ return identity;
391
+ }
392
+ function mergeRuntimeCandidateForExchange(left, right) {
393
+ return {
394
+ ...left,
395
+ ...right,
396
+ ...(right.eval ? { eval: right.eval } : left.eval ? { eval: left.eval } : {}),
397
+ ...(right.prompt ? { prompt: right.prompt } : left.prompt ? { prompt: left.prompt } : {}),
398
+ ...(right.meta !== undefined ? { meta: right.meta } : left.meta !== undefined ? { meta: left.meta } : {}),
399
+ ...(right.usage ? { usage: right.usage } : left.usage ? { usage: left.usage } : {}),
400
+ visibility: right.visibility ?? left.visibility,
401
+ };
402
+ }
403
+ function runtimeEvaluationsCompatibleForExchange(left, right) {
404
+ if (runtimeRecordsEqual(left, right)) {
405
+ return true;
406
+ }
407
+ return runtimeRecordsEqual(runtimeEvaluationIdentityForExchange(left), runtimeEvaluationIdentityForExchange(right));
408
+ }
409
+ function runtimeEvaluationIdentityForExchange(evaluation) {
410
+ return {
411
+ id: evaluation.id,
412
+ runId: evaluation.runId,
413
+ candidateId: evaluation.candidateId,
414
+ candidateVersion: evaluation.candidateVersion,
415
+ benchmarkFingerprint: evaluation.benchmarkFingerprint,
416
+ candidateFingerprint: evaluation.candidateFingerprint,
417
+ };
418
+ }
419
+ function runtimeRunsCompatibleForExchange(left, right) {
420
+ if (runtimeRecordsEqual(left, right)) {
421
+ return true;
422
+ }
423
+ return runtimeRecordsEqual(runtimeRunIdentityForExchange(left), runtimeRunIdentityForExchange(right));
424
+ }
425
+ function runtimeRunIdentityForExchange(run) {
426
+ return {
427
+ id: run.id,
428
+ workflow: run.workflow,
429
+ benchmarkFingerprint: run.benchmarkFingerprint,
430
+ candidateId: run.candidateId ?? null,
431
+ outputCandidateId: run.outputCandidateId ?? null,
432
+ engineRun: run.engineRun,
433
+ improver: run.improver,
434
+ strategy: run.strategy,
435
+ budget: run.budget,
436
+ samples: run.samples,
437
+ attemptsRequested: run.attemptsRequested,
438
+ };
439
+ }
440
+ function runtimeJobIdentityForExchange(job) {
441
+ return {
442
+ id: job.id,
443
+ runId: job.runId,
444
+ candidateId: job.candidateId,
445
+ kind: job.kind,
446
+ attempt: job.attempt,
447
+ };
448
+ }
449
+ function compatibleRuntimeActiveCandidateId(candidates, candidateId, benchmarkFingerprint) {
450
+ if (!candidateId) {
451
+ return null;
452
+ }
453
+ const candidate = candidates.find((entry) => entry.id === candidateId) ?? null;
454
+ return candidate?.benchmarkFingerprint === benchmarkFingerprint ? candidate.id : null;
455
+ }
456
+ function latestCompatibleRuntimeCandidateId(candidates, benchmarkFingerprint) {
457
+ return candidates
458
+ .filter((candidate) => candidate.benchmarkFingerprint === benchmarkFingerprint &&
459
+ candidate.status === "evaluated")
460
+ .at(-1)?.id ?? null;
461
+ }
372
462
  function canonicalRuntimeJson(value) {
373
463
  if (Array.isArray(value)) {
374
464
  return value.map(canonicalRuntimeJson);
@@ -431,6 +521,9 @@ function validateCandidateRecord(candidate) {
431
521
  requireArchivePositiveInteger(candidate.ordinal, `candidate ${candidate.id}.ordinal`);
432
522
  requireArchiveString(candidate.benchmarkFingerprint, `candidate ${candidate.id}.benchmarkFingerprint`);
433
523
  requireArchiveString(candidate.candidateFingerprint, `candidate ${candidate.id}.candidateFingerprint`);
524
+ if (candidate.visibility !== "private" && candidate.visibility !== "public") {
525
+ throw new Error(`candidate ${candidate.id}.visibility must be private or public.`);
526
+ }
434
527
  requireArchiveString(candidate.createdAt, `candidate ${candidate.id}.createdAt`);
435
528
  }
436
529
  function validateEvaluationRecord(evaluation) {
@@ -58,5 +58,6 @@ interface LocalProjectSourceOptions {
58
58
  }
59
59
  export declare function readLocalProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalProjectSource>;
60
60
  export declare function readLocalAuthoredProjectSource(source: string, options?: LocalProjectSourceOptions): Promise<LocalAuthoredProjectSource>;
61
+ export declare function hostedEngineResolveFiles(source: LocalProjectSource): HostedFile[];
61
62
  export {};
62
63
  //# sourceMappingURL=project-source.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"project-source.d.ts","sourceRoot":"","sources":["../src/project-source.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,kCAAkC,EAGlC,KAAK,IAAI,EACT,KAAK,mBAAmB,EACzB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAQL,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EAClC,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAGL,KAAK,qBAAqB,EAC3B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAML,KAAK,wBAAwB,EAC9B,MAAM,sBAAsB,CAAC;AAI9B,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAC5D,eAAO,MAAM,wBAAwB,eAAe,CAAC;AACrD,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC,OAAO,kCAAkC,CAAC,CAAC;IAC5D,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,UAAU,EAAE,CAAC;IAC9B,cAAc,EAAE,UAAU,EAAE,CAAC;IAC7B,kBAAkB,EAAE,UAAU,EAAE,CAAC;IACjC,QAAQ,EAAE,wBAAwB,EAAE,CAAC;IACrC,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,aAAa,EAAE,4BAA4B,CAAC;IAC5C,4BAA4B,EAAE,MAAM,CAAC;IACrC,wBAAwB,CAAC,EAAE,4BAA4B,CAAC,aAAa,CAAC,CAAC;IACvE,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,4BAA4B;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,CAAC,EAAE,IAAI,CAAC;CACb;AAED,UAAU,yBAAyB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAaD,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,kBAAkB,CAAC,CA8G7B;AAED,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,0BAA0B,CAAC,CA6BrC"}
1
+ {"version":3,"file":"project-source.d.ts","sourceRoot":"","sources":["../src/project-source.ts"],"names":[],"mappings":"AAKA,OAAO,EAOL,kCAAkC,EAGlC,KAAK,IAAI,EACT,KAAK,mBAAmB,EACzB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAUL,KAAK,mBAAmB,EACxB,KAAK,4BAA4B,EAClC,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAGL,KAAK,qBAAqB,EAC3B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAML,KAAK,wBAAwB,EAC9B,MAAM,sBAAsB,CAAC;AAI9B,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAC5D,eAAO,MAAM,wBAAwB,eAAe,CAAC;AACrD,eAAO,MAAM,wBAAwB,mBAAsB,CAAC;AAE5D,MAAM,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAE/C,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC,OAAO,kCAAkC,CAAC,CAAC;IAC5D,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,UAAU,EAAE,CAAC;IAC9B,cAAc,EAAE,UAAU,EAAE,CAAC;IAC7B,kBAAkB,EAAE,UAAU,EAAE,CAAC;IACjC,QAAQ,EAAE,wBAAwB,EAAE,CAAC;IACrC,YAAY,EAAE,UAAU,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,WAAW,EAAE,mBAAmB,EAAE,CAAC;IACnC,aAAa,EAAE,4BAA4B,CAAC;IAC5C,4BAA4B,EAAE,MAAM,CAAC;IACrC,wBAAwB,CAAC,EAAE,4BAA4B,CAAC,aAAa,CAAC,CAAC;IACvE,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,0BAA0B;IACzC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC;AAED,MAAM,WAAW,4BAA4B;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,CAAC,EAAE,IAAI,CAAC;CACb;AAED,UAAU,yBAAyB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAaD,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,kBAAkB,CAAC,CA8G7B;AAED,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,yBAA8B,GACtC,OAAO,CAAC,0BAA0B,CAAC,CA6BrC;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,kBAAkB,GAAG,UAAU,EAAE,CAqBjF"}