agent-hitch 0.2.6 → 0.2.7

package/dist/src/evals/planned-remote-execution.js

@@ -0,0 +1,40 @@
+import { preparedArtifactForWorkItem } from "./work-item-artifacts.js";
+import { loadTrialEnvironmentImages } from "./trial-environment-evidence.js";
+export async function executeRemotePlannedWorkItem(input) {
+    const { options, item } = input;
+    if (!options.remoteWorkExecutor)
+        throw new TypeError("remote work executor is unavailable");
+    const taskId = item.task_ids[0];
+    const environmentImages = await loadTrialEnvironmentImages({
+        taskId, uses: item.image_refs ?? [],
+        ...(options.environmentImageManifestLoader ? { loader: options.environmentImageManifestLoader } : {}),
+    });
+    const completed = await options.remoteWorkExecutor({
+        evalId: options.evalId,
+        evalDirectory: options.evalDirectory,
+        root: options.root,
+        request: options.request,
+        plan: options.plan,
+        workItem: item,
+        resolvedRevision: options.resolvedRevision,
+        preparedArtifact: preparedArtifactForWorkItem(options, item),
+        runtimeDirectory: options.controllerRuntime.directory,
+        runtimeId: options.controllerRuntime.runtime_id,
+        ...(environmentImages ? { environmentImages } : {}),
+        ...(options.plan.model_capture ? { modelCapturePlan: options.plan.model_capture } : {}),
+        ...(options.signal ? { signal: options.signal } : {}),
+        emit: (event) => options.sink.emit(event),
+        publish: input.publish,
+        onLeaseState: (leaseId, state) => options.onWorkItemState?.(item.work_id, leaseId, state) ?? Promise.resolve(),
+    });
+    return {
+        attempt: item.logical_attempt,
+        workId: item.work_id,
+        tasks: [taskId],
+        refs: completed.refs,
+        leaseId: completed.leaseId,
+        run: completed.run,
+        ...(environmentImages ? { environmentImages } : {}),
+    };
+}
+//# sourceMappingURL=planned-remote-execution.js.map

package/dist/src/evals/planned-remote-execution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"planned-remote-execution.js","sourceRoot":"","sources":["../../../src/evals/planned-remote-execution.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC;AACvE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAE7E,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,KAIlD;IACC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAChC,IAAI,CAAC,OAAO,CAAC,kBAAkB;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC5F,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,MAAM,iBAAiB,GAAG,MAAM,0BAA0B,CAAC;QACzD,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;QACnC,GAAG,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,8BAA8B,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtG,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,kBAAkB,CAAC;QACjD,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,IAAI;QACd,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,gBAAgB,EAAE,2BAA2B,CAAC,OAAO,EAAE,IAAI,CAAC;QAC5D,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,SAAS;QACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;QAC/C,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QACzC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,YAAY,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE;KAC/G,CAAC,CAAC;IACH,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,eAAyB;QACvC,MAAM,EAAE,IAAI,CAAC,OAAO;QACpB,KAAK,EAAE,CAAC,MAAM,CAAC;QACf,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,GAAG,EAAE,SAAS,CAAC,GAAG;QAClB,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpD,CAAC;AACJ,CAAC"}

package/dist/src/evals/planned-retry-lifecycle.js

@@ -0,0 +1,186 @@
+import { dockerResourceOwnership } from "./docker-ownership.js";
+import { startDockerResourceObserver } from "./docker-resource-observer.js";
+import { DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS, DEFAULT_EXECUTION_LEASE_TTL_MS, createExecutionLease } from "./execution-leases.js";
+import { resourceRequirementForTask, runtimeResourcesForTask } from "./execution-plan-resources.js";
+import { recordLocalDockerProcessExit, recordLocalDockerProcessStart, releaseLocalDockerProcessRecord } from "./local-docker-provider.js";
+import { resolvedImageMapping } from "./environment-image-planning.js";
+import { prebuiltTaskImage, verifyTrialEnvironmentImageExecution } from "./trial-environment-evidence.js";
+export async function beginPlannedInfrastructureRetry(input) {
+    const { options, item } = input;
+    const taskId = item.task_ids[0];
+    let permit;
+    let lease;
+    let runningAnnounced = false;
+    try {
+        permit = await options.admission?.acquire({
+            evalId: options.evalId,
+            workItem: item,
+            maxParallelism: options.plan.max_parallelism,
+            ...(options.signal ? { signal: options.signal } : {}),
+        });
+        if (permit)
+            options.sink.emit({
+                type: "eval.physical-retry.admitted",
+                execution_kind: "physical-infrastructure-retry",
+                work_id: item.work_id,
+                retry: input.retry,
+                allocation_id: permit.allocationId,
+                collision_keys: permit.collisionKeys,
+                reservation: item.reservation,
+            });
+        lease = await createExecutionLease({
+            evalDirectory: options.evalDirectory,
+            evalId: options.evalId,
+            workId: item.work_id,
+            worker: permit ? { ...options.worker, parentAllocationId: permit.allocationId } : options.worker,
+            reservation: item.reservation,
+            ttlMs: DEFAULT_EXECUTION_LEASE_TTL_MS,
+        });
+        const epoch = lease.current().epoch;
+        await lease.markRunning(epoch);
+        await options.onWorkItemState?.(item.work_id, lease.leaseId, "running");
+        runningAnnounced = true;
+        options.sink.emit({
+            type: "eval.physical-retry.started",
+            execution_kind: "physical-infrastructure-retry",
+            candidate_executes: true,
+            work_id: item.work_id,
+            lease_id: lease.leaseId,
+            lease_epoch: epoch,
+            retry: input.retry,
+            task_id: taskId,
+            attempt: item.logical_attempt,
+            trigger_trials: input.triggers.map((trial) => trial.trial_id).sort(),
+        });
+        return retryLifecycle(input, lease, permit, runningAnnounced);
+    }
+    catch (error) {
+        if (lease)
+            await lease.release(lease.current().epoch).catch(() => undefined);
+        if (runningAnnounced)
+            await options.onWorkItemState?.(item.work_id, lease?.leaseId ?? "", "terminal").catch(() => undefined);
+        permit?.release();
+        throw error;
+    }
+}
+function retryLifecycle(input, lease, permit, runningAnnounced) {
+    const { options, item } = input;
+    const taskId = item.task_ids[0];
+    const epoch = lease.current().epoch;
+    const runtime = runtimeResourcesForTask(options.plan, taskId, item.reservation);
+    const ownership = dockerResourceOwnership(options.root, lease.current(), taskId);
+    const observer = startDockerResourceObserver({
+        ownership,
+        workerId: options.worker.workerId,
+        collisionDomainId: options.worker.collisionDomainId,
+        reservation: item.reservation,
+        mainLimits: runtime.mainLimits,
+        sidecarLimits: runtime.sidecarLimits,
+        env: options.env,
+        ...(options.signal ? { signal: options.signal } : {}),
+    });
+    let processRecorded = false;
+    let heartbeatFailure;
+    let heartbeatTail = Promise.resolve();
+    const timer = setInterval(() => {
+        if (heartbeatFailure !== undefined)
+            return;
+        heartbeatTail = heartbeatTail.then(async () => {
+            const renewed = await lease.heartbeat(epoch);
+            options.sink.emit({
+                type: "lease.renewed",
+                execution_kind: "physical-infrastructure-retry",
+                work_id: item.work_id,
+                lease_id: lease.leaseId,
+                lease_epoch: renewed.epoch,
+                heartbeat_at: renewed.heartbeat_at,
+                expires_at: renewed.expires_at,
+            });
+        }).catch((error) => { heartbeatFailure ??= error; });
+    }, DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS);
+    timer.unref();
+    let closed;
+    const prebuiltImage = prebuiltTaskImage(input.environmentImages);
+    const close = () => closed ??= closeRetryLifecycle({ input, lease, permit, observer, timer, heartbeatTail: () => heartbeatTail, heartbeatFailure: () => heartbeatFailure, processRecorded: () => processRecorded, runningAnnounced });
+    return {
+        leaseId: lease.leaseId,
+        workId: item.work_id,
+        ...(input.environmentImages ? { environmentImages: input.environmentImages } : {}),
+        captureExecutionEvidence: async () => {
+            if (heartbeatFailure !== undefined)
+                throw heartbeatFailure;
+            return verifyTrialEnvironmentImageExecution(await observer.capture(), input.environmentImages);
+        },
+        backend: {
+            executionResources: resourceRequirementForTask(options.plan, taskId)?.main_limits ?? options.plan.default_trial_resources,
+            ...(Object.keys(runtime.sidecarLimits).length > 0 ? { dockerServiceLimits: runtime.sidecarLimits } : {}),
+            dockerOwnership: ownership,
+            resolvedImages: resolvedImageMapping(item.image_refs ?? []),
+            ...(prebuiltImage ? { prebuiltTaskImage: prebuiltImage } : {}),
+            ...(options.worker.provider === "local-docker" && process.platform !== "win32" ? {
+                recoverableProcess: true,
+                onProcessStarted: (pid) => recordLocalDockerProcessStart({
+                    root: options.root,
+                    workerId: options.worker.workerId,
+                    evalDirectory: options.evalDirectory,
+                    lease: lease.current(),
+                    backendDirectory: input.backendDirectory,
+                    pid,
+                }).then(() => { processRecorded = true; }),
+                onProcessExited: (result) => recordLocalDockerProcessExit({
+                    root: options.root,
+                    leaseId: lease.leaseId,
+                    epoch,
+                    ...result,
+                }).then(() => undefined),
+            } : {}),
+        },
+        close,
+    };
+}
+async function closeRetryLifecycle(input) {
+    const { options, item } = input.input;
+    const epoch = input.lease.current().epoch;
+    let failure;
+    const attempt = async (operation) => {
+        try {
+            await operation();
+        }
+        catch (error) {
+            failure ??= error;
+        }
+    };
+    clearInterval(input.timer);
+    await attempt(() => input.observer.stop());
+    await attempt(input.heartbeatTail);
+    failure ??= input.heartbeatFailure();
+    if (input.processRecorded())
+        await attempt(() => releaseLocalDockerProcessRecord({ root: options.root, leaseId: input.lease.leaseId, epoch }));
+    await attempt(async () => {
+        const released = await input.lease.release(epoch);
+        options.sink.emit({
+            type: "eval.physical-retry.lease-released",
+            execution_kind: "physical-infrastructure-retry",
+            work_id: item.work_id,
+            lease_id: input.lease.leaseId,
+            lease_epoch: released.epoch,
+            retry: input.input.retry,
+            state: released.state,
+        });
+    });
+    if (options.dockerResourceReaper) {
+        try {
+            const report = await options.dockerResourceReaper({ root: options.root, leaseIds: [input.lease.leaseId], env: options.env });
+            options.sink.emit({ type: "eval.physical-retry.resources-reaped", work_id: item.work_id, lease_id: input.lease.leaseId, retry: input.input.retry, scanned: report.scanned, deleted: report.deleted.length, issues: report.issues.length });
+        }
+        catch (error) {
+            options.sink.emit({ type: "eval.physical-retry.reaper-failed", work_id: item.work_id, lease_id: input.lease.leaseId, retry: input.input.retry, code: error.code || "docker_reaper_failed" });
+        }
+    }
+    if (input.runningAnnounced)
+        await attempt(() => options.onWorkItemState?.(item.work_id, input.lease.leaseId, "terminal") ?? Promise.resolve());
+    input.permit?.release();
+    if (failure !== undefined)
+        throw failure;
+}
+//# sourceMappingURL=planned-retry-lifecycle.js.map

package/dist/src/evals/planned-retry-lifecycle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"planned-retry-lifecycle.js","sourceRoot":"","sources":["../../../src/evals/planned-retry-lifecycle.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,2BAA2B,EAAE,MAAM,+BAA+B,CAAC;AAC5E,OAAO,EAAE,oCAAoC,EAAE,8BAA8B,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAEnI,OAAO,EAAE,0BAA0B,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAEpG,OAAO,EAAE,4BAA4B,EAAE,6BAA6B,EAAE,+BAA+B,EAAE,MAAM,4BAA4B,CAAC;AAE1I,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,oCAAoC,EAAE,MAAM,iCAAiC,CAAC;AAG1G,MAAM,CAAC,KAAK,UAAU,+BAA+B,CAAC,KAOrD;IACC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,IAAI,MAAyF,CAAC;IAC9F,IAAI,KAAuC,CAAC;IAC5C,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC;YACxC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,IAAI;YACd,cAAc,EAAE,OAAO,CAAC,IAAI,CAAC,eAAe;YAC5C,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC,CAAC;QACH,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC5B,IAAI,EAAE,8BAA8B;gBACpC,cAAc,EAAE,+BAA+B;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,aAAa,EAAE,MAAM,CAAC,YAAY;gBAClC,cAAc,EAAE,MAAM,CAAC,aAAa;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;QACH,KAAK,GAAG,MAAM,oBAAoB,CAAC;YACjC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM;YAChG,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,KAAK,EAAE,8BAA8B;SACtC,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;QACpC,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACxE,gBAAgB,GAAG,IAAI,CAAC;QACxB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,6BAA6B;YACnC,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,KAAK,CAAC,OAAO;YACvB,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,MAAM;YACf,OAAO,EAAE,IAAI,CAAC,eAAe;YAC7B,cAAc,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE;SACrE,CAAC,CAAC;QACH,OAAO,cAAc,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK;YAAE,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7E,IAAI,gBAAgB;YAAE,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC7H,MAAM,EAAE,OAAO,EAAE,CAAC;QAClB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACrB,KAMC,EACD,KAA2B,EAC3B,MAAyG,EACzG,gBAAyB;IAEzB,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;IACpC,MAAM,OAAO,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAChF,MAAM,SAAS,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,MAAM,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,2BAA2B,CAAC;QAC3C,SAAS;QACT,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;QACjC,iBAAiB,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB;QACnD,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtD,CAAC,CAAC;IACH,IAAI,eAAe,GAAG,KAAK,CAAC;IAC5B,IAAI,gBAAyB,CAAC;IAC9B,IAAI,aAAa,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,IAAI,gBAAgB,KAAK,SAAS;YAAE,OAAO;QAC3C,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAC5C,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,eAAe;gBACrB,cAAc,EAAE,+BAA+B;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,KAAK,CAAC,OAAO;gBACvB,WAAW,EAAE,OAAO,CAAC,KAAK;gBAC1B,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,UAAU,EAAE,OAAO,CAAC,UAAU;aAC/B,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACzC,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,IAAI,MAAiC,CAAC;IACtC,MAAM,aAAa,GAAG,iBAAiB,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,GAAkB,EAAE,CAAC,MAAM,KAAK,mBAAmB,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,aAAa,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,gBAAgB,EAAE,eAAe,EAAE,GAAG,EAAE,CAAC,eAAe,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACrP,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,IAAI,CAAC,OAAO;QACpB,GAAG,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,wBAAwB,EAAE,KAAK,IAAI,EAAE;YACnC,IAAI,gBAAgB,KAAK,SAAS;gBAAE,MAAM,gBAAgB,CAAC;YAC3D,OAAO,oCAAoC,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACjG,CAAC;QACD,OAAO,EAAE;YACP,kBAAkB,EAAE,0BAA0B,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,uBAAuB;YACzH,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACxG,eAAe,EAAE,SAAS;YAC1B,cAAc,EAAE,oBAAoB,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;YAC3D,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,cAAc,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC;gBAC/E,kBAAkB,EAAE,IAAI;gBACxB,gBAAgB,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,6BAA6B,CAAC;oBAC/D,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ;oBACjC,aAAa,EAAE,OAAO,CAAC,aAAa;oBACpC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE;oBACtB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;oBACxC,GAAG;iBACJ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC1C,eAAe,EAAE,CAAC,MAA8D,EAAE,EAAE,CAAC,4BAA4B,CAAC;oBAChH,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,KAAK;oBACL,GAAG,MAAM;iBACV,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;aACzB,CAAC,CAAC,CAAC,EAAE,CAAC;SACR;QACD,KAAK;KACN,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAUlC;IACC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC;IACtC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC;IAC1C,IAAI,OAAgB,CAAC;IACrB,MAAM,OAAO,GAAG,KAAK,EAAE,SAAiC,EAAiB,EAAE;QACzE,IAAI,CAAC;YAAC,MAAM,SAAS,EAAE,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAAC,OAAO,KAAK,KAAK,CAAC;QAAC,CAAC;IACjE,CAAC,CAAC;IACF,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3B,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACnC,OAAO,KAAK,KAAK,CAAC,gBAAgB,EAAE,CAAC;IACrC,IAAI,KAAK,CAAC,eAAe,EAAE;QAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,+BAA+B,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC/I,MAAM,OAAO,CAAC,KAAK,IAAI,EAAE;QACvB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,oCAAoC;YAC1C,cAAc,EAAE,+BAA+B;YAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO;YAC7B,WAAW,EAAE,QAAQ,CAAC,KAAK;YAC3B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK;YACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;QACjC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;YAC7H,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,sCAAsC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7O,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,mCAAmC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAG,KAA2B,CAAC,IAAI,IAAI,sBAAsB,EAAE,CAAC,CAAC;QACtN,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,gBAAgB;QAAE,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,UAAU,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/I,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;IACxB,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,OAAO,CAAC;AAC3C,CAAC"}

package/dist/src/evals/prepared-harness.js

@@ -0,0 +1,76 @@
+import { preparedArtifactDirectory } from "../artifacts/index.js";
+import { HitchError } from "../foundation/index.js";
+import { prepareHarness } from "../artifacts/index.js";
+export function harborPreparedArtifact(root, artifact) {
+    if (!artifact.artifact_integrity || !artifact.entrypoint_integrity) {
+        throw new HitchError("host-prepared harness artifact has no complete integrity metadata", {
+            code: "artifact_integrity_mismatch",
+            exitCode: 5,
+        });
+    }
+    return {
+        directory: preparedArtifactDirectory(root, artifact.artifact_id),
+        artifact_id: artifact.artifact_id,
+        artifact_integrity: artifact.artifact_integrity,
+        entrypoint_integrity: artifact.entrypoint_integrity,
+        harness_id: artifact.harness_id,
+        revision_identity: artifact.revision_identity,
+        adapter_version: artifact.adapter_version,
+        recipe_version: artifact.recipe_version,
+        platform: artifact.platform,
+        node_version: artifact.toolchain.node || process.version,
+        source_type: artifact.source_type,
+        storage: "host-artifact-store-v1",
+    };
+}
+export function preparedHarnessEvent(artifact, cacheHit, source, builder) {
+    return {
+        type: "eval.harness-artifact.prepared",
+        harness: artifact.harness_id,
+        artifact_id: artifact.artifact_id,
+        platform: artifact.platform,
+        cache_hit: cacheHit,
+        source,
+        ...(builder?.image ? { builder_image: builder.image } : {}),
+        ...(builder?.imageId ? { builder_image_id: builder.imageId } : {}),
+    };
+}
+export async function prepareEvalHarborArtifact(input) {
+    const selector = input.requestedReference.selector;
+    const verifiedLocalGitSource = input.localTransport && selector.type === "commit" && selector.source?.explicit
+        ? {
+            directory: selector.source.local_path,
+            commit: input.localTransport.manifest.commit,
+            tree: input.localTransport.manifest.tree,
+            resolutionIdentity: input.localTransport.manifest.resolution_identity,
+            payloadSha256: input.localTransport.manifest.payload_sha256,
+        }
+        : undefined;
+    return input.builder({
+        root: input.root,
+        resolvedRevision: input.resolvedRevision,
+        runtimeDirectory: input.controllerRuntime.directory,
+        runtimeId: input.controllerRuntime.runtime_id,
+        runtimeContract: input.runtimeContract,
+        ...(input.localTransport ? { localTransport: input.localTransport } : {}),
+        ...(verifiedLocalGitSource ? { verifiedLocalGitSource } : {}),
+        env: input.env,
+        ...(input.signal ? { signal: input.signal } : {}),
+    });
+}
+/** In-process seam for tests whose fake package manager cannot be mounted into
+ * Docker. Production callers must use the dedicated builder. */
+export const prepareHostHarborArtifactForTest = async (input) => {
+    const artifact = await prepareHarness(input.resolvedRevision, {
+        root: input.root,
+        env: input.env,
+        ...(input.signal ? { signal: input.signal } : {}),
+        ...(input.verifiedLocalGitSource ? { verifiedLocalGitSource: input.verifiedLocalGitSource } : {}),
+    });
+    return {
+        artifact: harborPreparedArtifact(input.root, artifact),
+        cacheHit: artifact.cache_hit,
+        source: "test-host",
+    };
+};
+//# sourceMappingURL=prepared-harness.js.map

package/dist/src/evals/prepared-harness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prepared-harness.js","sourceRoot":"","sources":["../../../src/evals/prepared-harness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAIlE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAKvD,MAAM,UAAU,sBAAsB,CAAC,IAAY,EAAE,QAA0B;IAC7E,IAAI,CAAC,QAAQ,CAAC,kBAAkB,IAAI,CAAC,QAAQ,CAAC,oBAAoB,EAAE,CAAC;QACnE,MAAM,IAAI,UAAU,CAAC,mEAAmE,EAAE;YACxF,IAAI,EAAE,6BAA6B;YACnC,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;IACL,CAAC;IACD,OAAO;QACL,SAAS,EAAE,yBAAyB,CAAC,IAAI,EAAE,QAAQ,CAAC,WAAW,CAAC;QAChE,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;QAC/C,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB;QACnD,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;QAC7C,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,YAAY,EAAE,QAAQ,CAAC,SAAS,CAAC,IAAI,IAAI,OAAO,CAAC,OAAO;QACxD,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,OAAO,EAAE,wBAAwB;KAClC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,QAA2E,EAC3E,QAAiB,EACjB,MAAyC,EACzC,OAA8C;IAE9C,OAAO;QACL,IAAI,EAAE,gCAAgC;QACtC,OAAO,EAAE,QAAQ,CAAC,UAAU;QAC5B,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,SAAS,EAAE,QAAQ;QACnB,MAAM;QACN,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,KAU/C;IACC,MAAM,QAAQ,GAAG,KAAK,CAAC,kBAAkB,CAAC,QAAQ,CAAC;IACnD,MAAM,sBAAsB,GAAG,KAAK,CAAC,cAAc,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,EAAE,QAAQ;QAC5G,CAAC,CAAC;YACE,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;YACrC,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM;YAC5C,IAAI,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,IAAI;YACxC,kBAAkB,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,mBAAmB;YACrE,aAAa,EAAE,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,cAAc;SAC5D;QACH,CAAC,CAAC,SAAS,CAAC;IACd,OAAO,KAAK,CAAC,OAAO,CAAC;QACnB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;QACxC,gBAAgB,EAAE,KAAK,CAAC,iBAAiB,CAAC,SAAS;QACnD,SAAS,EAAE,KAAK,CAAC,iBAAiB,CAAC,UAAU;QAC7C,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,GAAG,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CAAC,CAAC;AACL,CAAC;AAED;gEACgE;AAChE,MAAM,CAAC,MAAM,gCAAgC,GAA8B,KAAK,EAAE,KAAK,EAAE,EAAE;IACzF,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,gBAAgB,EAAE;QAC5D,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,KAAK,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClG,CAAC,CAAC;IACH,OAAO;QACL,QAAQ,EAAE,sBAAsB,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC;QACtD,QAAQ,EAAE,QAAQ,CAAC,SAAS;QAC5B,MAAM,EAAE,WAAW;KACpB,CAAC;AACJ,CAAC,CAAC"}

package/dist/src/evals/progress.js

@@ -32,7 +32,7 @@ export async function writeEvalProgress(evalDirectory, progress) {
     await atomicWriteJSON(path.join(evalDirectory, "progress.json"), parseEvalProgress(progress));
 }
 export function mergeEvalProgressTrial(progress, trial, now = new Date().toISOString()) {
-    const parsed = parseTrial(trial, "eval progress trial");
+    const parsed = parseEvalTrialRef(trial, "eval progress trial");
     const byTrial = progress.trials.find((item) => item.trial_id === parsed.trial_id);
     if (byTrial !== undefined) {
         if (JSON.stringify(byTrial) !== JSON.stringify(parsed))
@@ -66,7 +66,7 @@ export function evalTrialKey(trial) {
 }
 /** Replace one invalid logical trial, or fill a missing slot, with a valid verifier result. */
 export function replaceInvalidEvalProgressTrial(progress, trial, now = new Date().toISOString()) {
-    const parsed = parseTrial(trial, "eval rerun trial");
+    const parsed = parseEvalTrialRef(trial, "eval rerun trial");
     if (parsed.observation_status !== "valid")
         throw new TypeError("eval rerun replacement must be valid");
     const key = evalTrialKey(parsed);
@@ -118,7 +118,7 @@ export function parseEvalProgress(value) {
     }
     if (!Array.isArray(record.trials))
         throw new TypeError("eval progress trials are invalid");
-    const trials = record.trials.map((trial, index) => parseTrial(trial, `eval progress trial ${index}`));
+    const trials = record.trials.map((trial, index) => parseEvalTrialRef(trial, `eval progress trial ${index}`));
     if (new Set(trials.map((trial) => trial.trial_id)).size !== trials.length
         || new Set(trials.map((trial) => trial.run_id)).size !== trials.length)
         throw new TypeError("eval progress trial identities are duplicated");
@@ -164,7 +164,7 @@ export function parseEvalProgress(value) {
         updated_at: record.updated_at,
     };
 }
-function parseTrial(value, label) {
+export function parseEvalTrialRef(value, label = "eval trial") {
     if (!value || typeof value !== "object" || Array.isArray(value))
         throw new TypeError(`${label} must be an object`);
     const trial = value;

package/dist/src/evals/progress.js.map

@@ -1 +1 @@
-{"version":3,"file":"progress.js","sourceRoot":"","sources":["../../../src/evals/progress.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAEnE,MAAM,OAAO,GAAG,qBAAqB,CAAC;AACtC,MAAM,MAAM,GAAG,oBAAoB,CAAC;AAEpC,MAAM,UAAU,kBAAkB,CAAC,KAOlC;IACC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IACzF,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,iBAAiB;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IACvH,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC9G,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,YAAY,EAAE,KAAK,CAAC,WAAW;QAC/B,kBAAkB,EAAE,KAAK,CAAC,iBAAiB;QAC3C,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;QACzC,cAAc,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;QAC3C,MAAM,EAAE,EAAE;QACV,OAAO,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;QAClE,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,UAAU,EAAE,KAAK,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,aAAqB;IAC1D,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,CAAC;IAC9F,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,aAAqB,EAAE,QAAwB;IACrF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,QAAwB,EAAE,KAAqB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IACpH,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClF,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzI,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,SAAS,CAAC,wCAAwC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,SAAS,CAAC,yCAAyC,MAAM,CAAC,OAAO,YAAY,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3G,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACtG,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClF,OAAO,iBAAiB,CAAC;QACvB,GAAG,QAAQ;QACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC;QACnC,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkD;IAC7E,OAAO,GAAG,KAAK,CAAC,OAAO,SAAS,KAAK,CAAC,OAAO,EAAE,CAAC;AAClD,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,+BAA+B,CAC7C,QAAwB,EACxB,KAAqB,EACrB,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IAE9B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC;IACrD,IAAI,MAAM,CAAC,kBAAkB,KAAK,OAAO;QAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IACvG,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IAC5E,IAAI,QAAQ,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;QAC7C,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAAE,OAAO,QAAQ,CAAC;QACzE,MAAM,IAAI,SAAS,CAAC,yCAAyC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,SAAS,CAAC,wCAAwC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACpG,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACnJ,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClF,OAAO,iBAAiB,CAAC;QACvB,GAAG,QAAQ;QACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC;QACnC,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACxH,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACzG,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,YAAY;WAC9D,OAAO,MAAM,CAAC,kBAAkB,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,kBAAkB;WAC3E,MAAM,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,IAAI,SAAS,CAAC,2CAA2C,CAAC,CAAC;IACnG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,IAAK,MAAM,CAAC,UAAqB,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC9I,KAAK,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAU,EAAE,CAAC;QAChE,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,IAAK,OAAkB,GAAG,CAAC,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,iBAAiB,IAAI,aAAa,CAAC,CAAC;IAC/I,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC3F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,uBAAuB,KAAK,EAAE,CAAC,CAAC,CAAC;IACtG,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;WACpE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IAC/I,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACrF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAC;IACnI,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC/H,MAAM,aAAa,GAAG,OAAkC,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACpF,IAAI,MAAM,CAAC,cAAc,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,GAAI,MAAM,CAAC,cAAyB,EAAE,CAAC;QACxF,MAAM,IAAI,SAAS,CAAC,oDAAoD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAI,MAAM,CAAC,aAAwB,EAAE,CAAC;QAC3H,MAAM,IAAI,SAAS,CAAC,mDAAmD,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,aAAa,CAAC,cAAc,KAAK,MAAM,CAAC,MAAM,IAAI,aAAa,CAAC,YAAY,KAAK,KAAK;WACrF,aAAa,CAAC,cAAc,KAAK,MAAM,CAAC,MAAM,GAAG,KAAK;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IAChI,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;WACvF,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC7C,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,MAAM,CAAC,UAAoB;QACvC,aAAa,EAAE,MAAM,CAAC,aAA8B;QACpD,cAAc,EAAE,MAAM,CAAC,cAA+B;QACtD,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,KAAa;IAC/C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAC;IACnH,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ;WACpD,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;WAC9D,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO;WACnD,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAK,KAAK,CAAC,OAAkB,GAAG,CAAC;WACrE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,KAAK,CAAC,kBAAkB,KAAK,SAAS,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,CAAC,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACjH,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,0BAA0B,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,KAAK,CAAC,kBAAkB,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAClH,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,4BAA4B,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,mBAAmB,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC7H,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,0BAA0B,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAiB;QAChC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAgB,EAAE,CAAC;QACzE,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAA6B,EAAE,CAAC;QAChH,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAwB,EAAE,CAAC;KAClG,CAAC;AACJ,CAAC"}
+{"version":3,"file":"progress.js","sourceRoot":"","sources":["../../../src/evals/progress.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAEnE,MAAM,OAAO,GAAG,qBAAqB,CAAC;AACtC,MAAM,MAAM,GAAG,oBAAoB,CAAC;AAEpC,MAAM,UAAU,kBAAkB,CAAC,KAOlC;IACC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IACzF,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,iBAAiB;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IACvH,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC9G,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,KAAK,CAAC,MAAM;QACrB,YAAY,EAAE,KAAK,CAAC,WAAW;QAC/B,kBAAkB,EAAE,KAAK,CAAC,iBAAiB;QAC3C,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,KAAK,CAAC,YAAY,IAAI,IAAI;QACzC,cAAc,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;QAC3C,MAAM,EAAE,EAAE;QACV,OAAO,EAAE,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE;QAClE,UAAU,EAAE,KAAK,CAAC,SAAS;QAC3B,UAAU,EAAE,KAAK,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,aAAqB;IAC1D,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAiB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,CAAC;IAC9F,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,aAAqB,EAAE,QAAwB;IACrF,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,QAAwB,EAAE,KAAqB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IACpH,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;IAC/D,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,CAAC,CAAC;IAClF,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzI,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,SAAS,CAAC,wCAAwC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,SAAS,CAAC,yCAAyC,MAAM,CAAC,OAAO,YAAY,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3G,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACtG,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClF,OAAO,iBAAiB,CAAC;QACvB,GAAG,QAAQ;QACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC;QACnC,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkD;IAC7E,OAAO,GAAG,KAAK,CAAC,OAAO,SAAS,KAAK,CAAC,OAAO,EAAE,CAAC;AAClD,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,+BAA+B,CAC7C,QAAwB,EACxB,KAAqB,EACrB,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;IAE9B,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC;IAC5D,IAAI,MAAM,CAAC,kBAAkB,KAAK,OAAO;QAAE,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IACvG,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IAC5E,IAAI,QAAQ,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;QAC7C,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;YAAE,OAAO,QAAQ,CAAC;QACzE,MAAM,IAAI,SAAS,CAAC,yCAAyC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,SAAS,CAAC,wCAAwC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACpG,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACnJ,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IAClF,OAAO,iBAAiB,CAAC;QACvB,GAAG,QAAQ;QACX,UAAU,EAAE,QAAQ,CAAC,UAAU,GAAG,CAAC;QACnC,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc;IAC9C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACxH,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACzG,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IACD,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,YAAY;WAC9D,OAAO,MAAM,CAAC,kBAAkB,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,kBAAkB;WAC3E,MAAM,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,IAAI,SAAS,CAAC,2CAA2C,CAAC,CAAC;IACnG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,IAAK,MAAM,CAAC,UAAqB,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAC;IAC9I,KAAK,MAAM,IAAI,IAAI,CAAC,eAAe,EAAE,gBAAgB,CAAU,EAAE,CAAC;QAChE,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,OAAO,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,IAAK,OAAkB,GAAG,CAAC,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,iBAAiB,IAAI,aAAa,CAAC,CAAC;IAC/I,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC3F,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,EAAE,uBAAuB,KAAK,EAAE,CAAC,CAAC,CAAC;IAC7G,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;WACpE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,+CAA+C,CAAC,CAAC;IAC/I,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;WACrF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO;WAC5B,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAClD,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,iDAAiD,CAAC,CAAC;IACnI,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC/H,MAAM,aAAa,GAAG,OAAkC,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;IACpF,IAAI,MAAM,CAAC,cAAc,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,GAAI,MAAM,CAAC,cAAyB,EAAE,CAAC;QACxF,MAAM,IAAI,SAAS,CAAC,oDAAoD,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,MAAM,CAAC,aAAa,KAAK,IAAI,IAAI,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAI,MAAM,CAAC,aAAwB,EAAE,CAAC;QAC3H,MAAM,IAAI,SAAS,CAAC,mDAAmD,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,aAAa,CAAC,cAAc,KAAK,MAAM,CAAC,MAAM,IAAI,aAAa,CAAC,YAAY,KAAK,KAAK;WACrF,aAAa,CAAC,cAAc,KAAK,MAAM,CAAC,MAAM,GAAG,KAAK;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IAChI,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;WACvF,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,IAAI,SAAS,CAAC,sCAAsC,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO;QACL,cAAc,EAAE,GAAG;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC7C,MAAM,EAAE,SAAS;QACjB,UAAU,EAAE,MAAM,CAAC,UAAoB;QACvC,aAAa,EAAE,MAAM,CAAC,aAA8B;QACpD,cAAc,EAAE,MAAM,CAAC,cAA+B;QACtD,MAAM;QACN,OAAO,EAAE;YACP,cAAc,EAAE,MAAM,CAAC,MAAM;YAC7B,YAAY,EAAE,KAAK;YACnB,cAAc,EAAE,MAAM,CAAC,MAAM,GAAG,KAAK;SACtC;QACD,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAc,EAAE,KAAK,GAAG,YAAY;IACpE,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAC;IACnH,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,QAAQ;WACpD,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;WAC9D,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO;WACnD,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,IAAK,KAAK,CAAC,OAAkB,GAAG,CAAC;WACrE,CAAC,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,KAAK,CAAC,kBAAkB,KAAK,SAAS,CAAC,EAAE,CAAC;QACtF,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,KAAK,CAAC,kBAAkB,KAAK,OAAO,IAAI,CAAC,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACjH,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,0BAA0B,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,KAAK,CAAC,kBAAkB,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAClH,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,4BAA4B,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,KAAK,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAC,OAAO,KAAK,CAAC,mBAAmB,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC7H,MAAM,IAAI,SAAS,CAAC,GAAG,KAAK,0BAA0B,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO;QACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAiB;QAChC,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAgB,EAAE,CAAC;QACzE,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAA6B,EAAE,CAAC;QAChH,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAwB,EAAE,CAAC;KAClG,CAAC;AACJ,CAAC"}

package/dist/src/evals/records.js

@@ -1,7 +1,8 @@
-import { readdir } from "node:fs/promises";
+import { lstat, readdir } from "node:fs/promises";
 import path from "node:path";
 import { inspectEvalRuntimeKind } from "../controller-runtime/index.js";
 import { HitchError, SCHEMA_VERSION, invalidInput, readJSON, statePaths } from "../foundation/index.js";
+import { readExecutionLeases } from "./execution-leases.js";
 export async function listEvals({ root }) {
     const directory = statePaths(root).evals;
     let entries;
@@ -19,16 +20,17 @@ export async function listEvals({ root }) {
             continue;
         const result = await readJSON(path.join(directory, entry.name, "result.json"), null).catch(() => null);
         const request = await readJSON(path.join(directory, entry.name, "request.json"), null).catch(() => null);
+        const control = await readJSON(path.join(directory, entry.name, "control.json"), null).catch(() => null);
         if (!result && !request)
             continue;
         evals.push({
             eval_id: entry.name,
-            status: result?.status || "running",
+            status: result?.status || control?.state || "running",
             backend: result?.backend?.name || request?.backend || null,
             dataset: result?.dataset || request?.dataset || null,
             harness_ref: result?.candidate?.harness_ref || request?.harness_ref || null,
             primary_reward: result?.summary?.primary_reward ?? null,
-            started_at: result?.started_at || null,
+            started_at: result?.started_at || control?.created_at || null,
             completed_at: result?.completed_at || null,
         });
     }
@@ -48,8 +50,23 @@ export async function inspectEval(evalId, { root }) {
         request,
         resolution: await readJSON(path.join(directory, "resolution.json"), null),
         plan: await readJSON(path.join(directory, "plan.json"), null),
+        execution_plan: await readJSON(path.join(directory, "execution-plan.json"), null),
+        submission: await readJSON(path.join(directory, "submission.json"), null),
+        control: await readJSON(path.join(directory, "control.json"), null),
+        progress: await readJSON(path.join(directory, "progress.json"), null),
+        leases: await readExecutionLeases(directory),
         result: await readJSON(path.join(directory, "result.json"), null),
         runtime_storage: await inspectEvalRuntimeKind(directory),
     };
 }
+export async function isControlPlaneEval(root, evalId) {
+    try {
+        return (await lstat(path.join(statePaths(root).evals, evalId, "submission.json"))).isFile();
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return false;
+        throw error;
+    }
+}
 //# sourceMappingURL=records.js.map

package/dist/src/evals/records.js.map

@@ -1 +1 @@
-{"version":3,"file":"records.js","sourceRoot":"","sources":["../../../src/evals/records.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAaxG,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,EAAE,IAAI,EAAoB;IACxD,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;IACzC,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,EAAE,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QACnE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAS;QACtE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACvI,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACzI,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;YAAE,SAAS;QAClC,KAAK,CAAC,IAAI,CAAC;YACT,OAAO,EAAE,KAAK,CAAC,IAAI;YACnB,MAAM,EAAG,MAAM,EAAE,MAAiB,IAAI,SAAS;YAC/C,OAAO,EAAI,MAAM,EAAE,OAAmC,EAAE,IAAe,IAAK,OAAO,EAAE,OAAkB,IAAI,IAAI;YAC/G,OAAO,EAAG,MAAM,EAAE,OAAkB,IAAK,OAAO,EAAE,OAAkB,IAAI,IAAI;YAC5E,WAAW,EAAI,MAAM,EAAE,SAAqC,EAAE,WAAsB,IAAK,OAAO,EAAE,WAAsB,IAAI,IAAI;YAChI,cAAc,EAAG,MAAM,EAAE,OAAmC,EAAE,cAA+B,IAAI,IAAI;YACrG,UAAU,EAAG,MAAM,EAAE,UAAqB,IAAI,IAAI;YAClD,YAAY,EAAG,MAAM,EAAE,YAAuB,IAAI,IAAI;SACvD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACvI,CAAC;AAaD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc,EAAE,EAAE,IAAI,EAAoB;IAC1E,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,MAAM,YAAY,CAAC,oBAAoB,MAAM,EAAE,CAAC,CAAC;IACxH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;IAC3G,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,UAAU,CAAC,mBAAmB,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IACzG,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,MAAM;QACf,SAAS;QACT,OAAO;QACP,UAAU,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,IAAI,CAAC;QACzG,IAAI,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC;QAC7F,MAAM,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC;QACjG,eAAe,EAAE,MAAM,sBAAsB,CAAC,SAAS,CAAC;KACzD,CAAC;AACJ,CAAC"}
+{"version":3,"file":"records.js","sourceRoot":"","sources":["../../../src/evals/records.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,sBAAsB,EAAE,MAAM,gCAAgC,CAAC;AAExE,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACxG,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAa5D,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,EAAE,IAAI,EAAoB;IACxD,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;IACzC,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,EAAE,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QACnE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAS;QACtE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACvI,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACzI,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACzI,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO;YAAE,SAAS;QAClC,KAAK,CAAC,IAAI,CAAC;YACT,OAAO,EAAE,KAAK,CAAC,IAAI;YACnB,MAAM,EAAG,MAAM,EAAE,MAAiB,IAAK,OAAO,EAAE,KAAgB,IAAI,SAAS;YAC7E,OAAO,EAAI,MAAM,EAAE,OAAmC,EAAE,IAAe,IAAK,OAAO,EAAE,OAAkB,IAAI,IAAI;YAC/G,OAAO,EAAG,MAAM,EAAE,OAAkB,IAAK,OAAO,EAAE,OAAkB,IAAI,IAAI;YAC5E,WAAW,EAAI,MAAM,EAAE,SAAqC,EAAE,WAAsB,IAAK,OAAO,EAAE,WAAsB,IAAI,IAAI;YAChI,cAAc,EAAG,MAAM,EAAE,OAAmC,EAAE,cAA+B,IAAI,IAAI;YACrG,UAAU,EAAG,MAAM,EAAE,UAAqB,IAAK,OAAO,EAAE,UAAqB,IAAI,IAAI;YACrF,YAAY,EAAG,MAAM,EAAE,YAAuB,IAAI,IAAI;SACvD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACvI,CAAC;AAkBD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc,EAAE,EAAE,IAAI,EAAoB;IAC1E,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,MAAM,YAAY,CAAC,oBAAoB,MAAM,EAAE,CAAC,CAAC;IACxH,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,CAAC;IAC3G,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,UAAU,CAAC,mBAAmB,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IACzG,OAAO;QACL,cAAc,EAAE,cAAc;QAC9B,OAAO,EAAE,MAAM;QACf,SAAS;QACT,OAAO;QACP,UAAU,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,IAAI,CAAC;QACzG,IAAI,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC;QAC7F,cAAc,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,EAAE,IAAI,CAAC;QACjH,UAAU,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iBAAiB,CAAC,EAAE,IAAI,CAAC;QACzG,OAAO,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC;QACnG,QAAQ,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC;QACrG,MAAM,EAAE,MAAM,mBAAmB,CAAC,SAAS,CAAC;QAC5C,MAAM,EAAE,MAAM,QAAQ,CAAiC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,IAAI,CAAC;QACjG,eAAe,EAAE,MAAM,sBAAsB,CAAC,SAAS,CAAC;KACzD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY,EAAE,MAAc;IACnE,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC9F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QACrE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}