@bjoernrochel/sevro 1.0.0 → 1.1.1

package/dist/src/evaluation/plan.js

@@ -0,0 +1,376 @@
+import { realpath } from "node:fs/promises";
+import { EvaluationConfigurationError, } from "./types.js";
+import { prepareFixtureSetup } from "../fixtures/setup.js";
+import { canonicalJson, createEvaluationIdentity, hashJson } from "../identity.js";
+import { prepareInstrumentation, } from "../instrumentation.js";
+import { fixtureParts, prepareArtifacts, } from "../fixtures/preparation.js";
+import { projectProvenance, runnerProvenance } from "../provenance.js";
+import { resolveRepositorySource } from "../fixtures/repository.js";
+import { prepareEvaluationChecks, } from "./check-plan.js";
+import { prepareInvocationPlan, } from "./invocation-plan.js";
+async function prepareEvaluationEnvironment(options, source) {
+    const projectRoot = await realpath(options.projectRoot).catch(() => {
+        throw new EvaluationConfigurationError("project root is unreadable");
+    });
+    const resolvedSource = await resolveFixtureSource(options, source);
+    const [runner, project] = await Promise.all([
+        runnerProvenance(options.runnerBuildDigest, options.runnerCheckoutRoot).catch((error) => {
+            throw new EvaluationConfigurationError(error instanceof Error ? error.message : "invalid runner provenance");
+        }),
+        projectProvenance(projectRoot, evaluationStorageRoots(options)),
+    ]);
+    const extensionPreparation = options.extension
+        ? await options.extension.session.prepare(options.extension.resolvedCase, {
+            id: options.host.id,
+            capabilities: [
+                ...(options.host.instrumentation ?? []).map((item) => item.id),
+                ...(options.host.hostCapabilities ?? []),
+            ],
+        }, options.condition)
+        : null;
+    return {
+        projectRoot,
+        source: resolvedSource,
+        runner,
+        project,
+        extensionPreparation,
+    };
+}
+async function resolveFixtureSource(options, source) {
+    if (source.kind !== "repository")
+        return source;
+    const repository = await resolveRepositorySource(source.fixture.sourceRef, options.preparationSources).catch((error) => {
+        throw new EvaluationConfigurationError(error instanceof Error ? error.message : "invalid repository source");
+    });
+    return { ...source, repository };
+}
+function evaluationStorageRoots(options) {
+    return [options.resultsRoot, options.runStateRoot ?? options.resultsRoot];
+}
+function preparedFixtureSetup(preparation) {
+    try {
+        return prepareFixtureSetup(preparation?.fixtureSetup);
+    }
+    catch (cause) {
+        throw new EvaluationConfigurationError(cause instanceof Error ? cause.message : "invalid fixture setup", { cause });
+    }
+}
+function validateSetupCapability(options, setup) {
+    if (setup &&
+        !options.extension?.session.identity.capabilities.includes("sevro.fixture.setup"))
+        throw new EvaluationConfigurationError("fixture setup capability was not negotiated");
+}
+function validateSetupFixture(setup, source) {
+    if (setup && source.kind === "inline")
+        throw new EvaluationConfigurationError("fixture setup requires a Git fixture");
+}
+function negotiatedInstrumentationCapabilities(options) {
+    return options.extension?.session.identity.capabilities ?? [];
+}
+function configurationFailureMessage(cause, fallback) {
+    return cause instanceof Error ? cause.message : fallback;
+}
+function requestedInstrumentationPlan(options, preparation) {
+    try {
+        return prepareInstrumentation(preparation?.requestedInstrumentation ?? [], options.host.instrumentation ?? [], negotiatedInstrumentationCapabilities(options), options.condition);
+    }
+    catch (cause) {
+        throw new EvaluationConfigurationError(configurationFailureMessage(cause, "invalid instrumentation"), { cause });
+    }
+}
+function fixturePreparationPaths(source) {
+    switch (source.kind) {
+        case "inline":
+            return Object.keys(source.files ?? {});
+        case "repository":
+            return Object.keys(source.fixture.files ?? {});
+        case "generated":
+            return [
+                ...new Set([
+                    ...source.fixture.commits.flatMap((commit) => Object.keys(commit.files)),
+                    ...Object.keys(source.fixture.files ?? {}),
+                ]),
+            ];
+    }
+}
+async function preparedEvaluationArtifacts(options, context) {
+    try {
+        return await prepareArtifacts(context.extensionPreparation?.artifacts ?? [], fixturePreparationPaths(context.source), options.preparationSources);
+    }
+    catch (cause) {
+        throw new EvaluationConfigurationError(cause instanceof Error ? cause.message : "invalid preparation artifacts", { cause });
+    }
+}
+function validateArtifactMetadata(artifacts, source) {
+    if (source.kind !== "inline" &&
+        artifacts.some((artifact) => fixtureParts(artifact.relativePath).some((part) => part.toLowerCase() === ".git")))
+        throw new EvaluationConfigurationError("preparation artifacts cannot modify repository metadata");
+}
+function validateArtifactExclusions(artifacts, source) {
+    if (source.kind === "inline" &&
+        artifacts.some((artifact) => artifact.gitExclude))
+        throw new EvaluationConfigurationError("Git-excluded preparation artifacts require a Git fixture");
+}
+function validateSemanticArtifactId(artifacts, declarations) {
+    if (declarations.length &&
+        artifacts.some((item) => item.id === "sevro.semantic.verdicts"))
+        throw new EvaluationConfigurationError("preparation artifact uses a reserved semantic evidence ID");
+}
+async function prepareEvaluationArtifacts(options, state, checks) {
+    const fixtureSetup = preparedFixtureSetup(state.extensionPreparation);
+    validateSetupCapability(options, fixtureSetup);
+    validateSetupFixture(fixtureSetup, state.source);
+    const requestedInstrumentation = requestedInstrumentationPlan(options, state.extensionPreparation);
+    const inlineArtifacts = await preparedEvaluationArtifacts(options, state);
+    validateArtifactMetadata(inlineArtifacts, state.source);
+    validateArtifactExclusions(inlineArtifacts, state.source);
+    validateSemanticArtifactId(inlineArtifacts, checks.checks.semanticDeclarations);
+    return { fixtureSetup, requestedInstrumentation, inlineArtifacts };
+}
+function extensionPreparationData(options, preparation) {
+    return options.extension
+        ? {
+            ...options.extension.resolvedCase.extensionData,
+            ...preparation?.extensionData,
+        }
+        : {};
+}
+function evaluationRoute(host, role) {
+    return { role, host: host.id, model: host.model, effort: host.effort };
+}
+function semanticEvaluationRoute(host) {
+    if (!host)
+        throw new EvaluationConfigurationError("semantic checks require an explicit semantic host");
+    return evaluationRoute(host, "semantic");
+}
+function evaluationRoutes(options, semantic, candidate) {
+    return [
+        candidate,
+        ...(semantic ? [semanticEvaluationRoute(options.semanticHost)] : []),
+        ...(options.advisoryHost
+            ? [evaluationRoute(options.advisoryHost, "advisory")]
+            : []),
+    ];
+}
+function redactedHostConfiguration(options, semantic) {
+    const selections = [
+        { role: "candidate", host: options.host },
+        { role: "semantic", host: semantic ? options.semanticHost : undefined },
+        { role: "advisory", host: options.advisoryHost },
+    ];
+    const configuration = Object.fromEntries(selections.flatMap(({ role, host }) => {
+        if (host?.configuration === undefined)
+            return [];
+        return [[role, host.configuration]];
+    }));
+    return Object.keys(configuration).length
+        ? { hostConfiguration: configuration }
+        : {};
+}
+function redactedExtensionConfiguration(options) {
+    return {
+        extensionConfigurationDigest: options.extension?.session.identity.configurationDigest ?? null,
+        ...(options.extension
+            ? {
+                extensionConfiguration: options.extension.session.redactedConfiguration,
+            }
+            : {}),
+    };
+}
+function redactedPreparationConfiguration(state) {
+    return {
+        ...(state.fixture.fixtureSetup
+            ? { fixtureSetupDigest: hashJson(state.fixture.fixtureSetup) }
+            : {}),
+        ...(state.invocation.codexMarketplace
+            ? { codexMarketplace: state.invocation.codexMarketplace }
+            : {}),
+        ...(state.invocation.claudePluginDirs
+            ? { claudePluginDirs: state.invocation.claudePluginDirs }
+            : {}),
+    };
+}
+function redactedAdvisoryConfiguration(options) {
+    return options.advisoryHost
+        ? {
+            advisoryExcludedPaths: [
+                ...(options.advisoryExcludedPaths ?? []),
+            ].sort(),
+        }
+        : {};
+}
+function redactedEvaluationConfiguration(options, state) {
+    return {
+        condition: options.condition,
+        executionMode: options.dry ? "dry" : "executed",
+        ...(options.runtimePolicy ? { runtimePolicy: options.runtimePolicy } : {}),
+        trialCount: options.trialCount,
+        jobs: state.jobs,
+        passThreshold: options.passThreshold,
+        ...(options.assessmentSettings
+            ? { runSettings: options.assessmentSettings }
+            : {}),
+        ...redactedHostConfiguration(options, state.checks.preparedSemantic.length > 0),
+        ...redactedExtensionConfiguration(options),
+        ...redactedPreparationConfiguration(state),
+        ...state.invocation.configuration,
+        ...redactedAdvisoryConfiguration(options),
+    };
+}
+function activeEvaluationGraders(options, state) {
+    const graders = [
+        ...new Set([
+            ...state.builtinDeclarations,
+            ...state.shellDeclarations,
+            ...state.gitHeadDeclarations,
+            ...state.semanticDeclarations,
+        ].map((check) => check.grader)),
+    ]
+        .sort()
+        .map((id) => ({ id, source: "builtin", version: "1.0.0" }));
+    if (options.extension) {
+        for (const id of [
+            ...new Set(state.extensionDeclarations.map((check) => check.grader)),
+        ].sort())
+            graders.push({
+                id,
+                source: "extension",
+                version: options.extension.session.identity.version,
+            });
+    }
+    return graders;
+}
+function extensionIdentityDimensions(options) {
+    return {
+        extensionDigest: options.extension?.session.identity.sourceDigest ?? null,
+        extensionProtocol: options.extension?.session.identity.protocol ?? null,
+    };
+}
+function caseIdentityData(caseData, extensionData) {
+    return {
+        id: caseData.id,
+        prompt: caseData.prompt,
+        ...(caseData.followUpPrompt !== undefined
+            ? { followUpPrompt: caseData.followUpPrompt }
+            : {}),
+        extensionData,
+    };
+}
+function baseFixtureIdentity(source) {
+    switch (source.kind) {
+        case "generated":
+            return source.fixture;
+        case "repository":
+            return { ...source.fixture, revision: source.repository.revision };
+        case "inline":
+            return { files: source.files };
+    }
+}
+function artifactIdentityData(artifacts) {
+    return artifacts.map(({ id, relativePath, sha256, gitExclude, executable }) => ({
+        id,
+        relativePath,
+        sha256,
+        ...(gitExclude ? { gitExclude: true } : {}),
+        ...(executable ? { executable: true } : {}),
+    }));
+}
+function fixtureIdentityData(state) {
+    return {
+        ...baseFixtureIdentity(state.fixture.source),
+        artifacts: artifactIdentityData(state.fixture.inlineArtifacts),
+        ...(state.fixture.fixtureSetup
+            ? { fixtureSetup: state.fixture.fixtureSetup }
+            : {}),
+        ...(state.invocation.codexMarketplace
+            ? { codexMarketplace: state.invocation.codexMarketplace }
+            : {}),
+        ...(state.invocation.claudePluginDirs
+            ? { claudePluginDirs: state.invocation.claudePluginDirs }
+            : {}),
+        ...state.invocation.configuration,
+    };
+}
+function evaluatorIdentityData(options, extensionData) {
+    return {
+        policy: "sevro.builtin-output.v1",
+        extension: options.extension?.session.identity ?? null,
+        extensionData,
+    };
+}
+function evaluationComparisonIdentity(options, prepared, context) {
+    try {
+        return createEvaluationIdentity({
+            runnerBuildDigest: options.runnerBuildDigest,
+            projectDigest: options.projectDigest,
+            configurationDigest: hashJson(context.redactedConfig),
+            ...extensionIdentityDimensions(options),
+            caseDigest: hashJson(caseIdentityData(options.case, context.extensionData)),
+            fixtureDigest: hashJson(fixtureIdentityData(prepared)),
+            checksDigest: hashJson(options.case.checks),
+            requiredEvidenceDigest: hashJson(options.case.requiredEvidence),
+            evaluatorDigest: hashJson(evaluatorIdentityData(options, context.extensionData)),
+            graderDigest: hashJson(context.activeGraders),
+            instrumentationDigest: hashJson({
+                requested: prepared.requestedInstrumentation,
+                applied: prepared.requestedInstrumentation,
+            }),
+            routeDigest: hashJson(context.routes.length > 1 ? context.routes : context.route),
+            condition: options.condition,
+            trialCount: options.trialCount,
+            passThreshold: options.passThreshold,
+        });
+    }
+    catch (cause) {
+        throw new EvaluationConfigurationError("invalid evaluation identity inputs", { cause });
+    }
+}
+function prepareEvaluationIdentity(options, state, preparation) {
+    const extensionData = extensionPreparationData(options, preparation);
+    const route = evaluationRoute(options.host, "candidate");
+    const routes = evaluationRoutes(options, state.checks.preparedSemantic.length > 0, route);
+    const redactedConfig = snapshotEvaluationConfiguration(options, state);
+    const activeGraders = activeEvaluationGraders(options, state.checks);
+    const context = {
+        extensionData,
+        route,
+        routes,
+        redactedConfig,
+        activeGraders,
+    };
+    const evaluationIdentity = evaluationComparisonIdentity(options, state, context);
+    return { ...state, ...context, evaluationIdentity };
+}
+function snapshotEvaluationConfiguration(options, state) {
+    try {
+        return JSON.parse(canonicalJson(redactedEvaluationConfiguration(options, state)));
+    }
+    catch (cause) {
+        throw new EvaluationConfigurationError("invalid evaluation identity inputs", { cause });
+    }
+}
+export async function prepareEvaluation(options) {
+    const checks = prepareEvaluationChecks(options);
+    const environment = await prepareEvaluationEnvironment(options, checks.source);
+    const artifacts = await prepareEvaluationArtifacts(options, environment, checks);
+    const invocation = prepareInvocationPlan(options, {
+        extensionPreparation: environment.extensionPreparation,
+        inlineArtifacts: artifacts.inlineArtifacts,
+    });
+    const prepared = {
+        fixture: {
+            projectRoot: environment.projectRoot,
+            source: environment.source,
+            fixtureSetup: artifacts.fixtureSetup,
+            inlineArtifacts: artifacts.inlineArtifacts,
+        },
+        checks: checks.checks,
+        invocation,
+        jobs: checks.jobs,
+        requestedInstrumentation: artifacts.requestedInstrumentation,
+        runner: environment.runner,
+        project: environment.project,
+    };
+    return prepareEvaluationIdentity(options, prepared, environment.extensionPreparation);
+}
+//# sourceMappingURL=plan.js.map

package/dist/src/evaluation/plan.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"plan.js","sourceRoot":"","sources":["plan.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,yBAAyB;AAC5C,OAAO,EACL,4BAA4B,GAI7B,mBAAgB;AACjB,OAAO,EAAE,mBAAmB,EAAqB,6BAA0B;AAG3E,OAAO,EAAE,aAAa,EAAE,wBAAwB,EAAE,QAAQ,EAAE,uBAAoB;AAChF,OAAO,EACL,sBAAsB,GAEvB,8BAA2B;AAC5B,OAAO,EACL,YAAY,EACZ,gBAAgB,GAEjB,mCAAgC;AACjC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,yBAAsB;AACpE,OAAO,EAAE,uBAAuB,EAAE,kCAA+B;AACjE,OAAO,EACL,uBAAuB,GAGxB,wBAAqB;AACtB,OAAO,EACL,qBAAqB,GAEtB,6BAA0B;AAoB3B,KAAK,UAAU,4BAA4B,CACzC,OAA0B,EAC1B,MAA2B;IAE3B,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE;QACjE,MAAM,IAAI,4BAA4B,CAAC,4BAA4B,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IACH,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACnE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC1C,gBAAgB,CACd,OAAO,CAAC,iBAAiB,EACzB,OAAO,CAAC,kBAAkB,CAC3B,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YACzB,MAAM,IAAI,4BAA4B,CACpC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,CACrE,CAAC;QACJ,CAAC,CAAC;QACF,iBAAiB,CAAC,WAAW,EAAE,sBAAsB,CAAC,OAAO,CAAC,CAAC;KAChE,CAAC,CAAC;IACH,MAAM,oBAAoB,GAAG,OAAO,CAAC,SAAS;QAC5C,CAAC,CAAC,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CACrC,OAAO,CAAC,SAAS,CAAC,YAAY,EAC9B;YACE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE;YACnB,YAAY,EAAE;gBACZ,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9D,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,IAAI,EAAE,CAAC;aACzC;SACF,EACD,OAAO,CAAC,SAAS,CAClB;QACH,CAAC,CAAC,IAAI,CAAC;IACT,OAAO;QACL,WAAW;QACX,MAAM,EAAE,cAAc;QACtB,MAAM;QACN,OAAO;QACP,oBAAoB;KACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,OAA0B,EAC1B,MAA2B;IAE3B,IAAI,MAAM,CAAC,IAAI,KAAK,YAAY;QAAE,OAAO,MAAM,CAAC;IAChD,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAC9C,MAAM,CAAC,OAAO,CAAC,SAAS,EACxB,OAAO,CAAC,kBAAkB,CAC3B,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QACzB,MAAM,IAAI,4BAA4B,CACpC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,CACrE,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,sBAAsB,CAAC,OAA0B;IACxD,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,oBAAoB,CAC3B,WAAoD;IAEpD,IAAI,CAAC;QACH,OAAO,mBAAmB,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,4BAA4B,CACpC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,EAChE,EAAE,KAAK,EAAE,CACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAC9B,OAA0B,EAC1B,KAA0B;IAE1B,IACE,KAAK;QACL,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,QAAQ,CACxD,qBAAqB,CACtB;QAED,MAAM,IAAI,4BAA4B,CACpC,6CAA6C,CAC9C,CAAC;AACN,CAAC;AAED,SAAS,oBAAoB,CAC3B,KAA0B,EAC1B,MAAqB;IAErB,IAAI,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ;QACnC,MAAM,IAAI,4BAA4B,CACpC,sCAAsC,CACvC,CAAC;AACN,CAAC;AAED,SAAS,qCAAqC,CAC5C,OAA0B;IAE1B,OAAO,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,SAAS,2BAA2B,CAAC,KAAc,EAAE,QAAgB;IACnE,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC3D,CAAC;AAED,SAAS,4BAA4B,CACnC,OAA0B,EAC1B,WAAoD;IAEpD,IAAI,CAAC;QACH,OAAO,sBAAsB,CAC3B,WAAW,EAAE,wBAAwB,IAAI,EAAE,EAC3C,OAAO,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,EAClC,qCAAqC,CAAC,OAAO,CAAC,EAC9C,OAAO,CAAC,SAAS,CAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,4BAA4B,CACpC,2BAA2B,CAAC,KAAK,EAAE,yBAAyB,CAAC,EAC7D,EAAE,KAAK,EAAE,CACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,MAAqB;IACpD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,QAAQ;YACX,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACzC,KAAK,YAAY;YACf,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;QACjD,KAAK,WAAW;YACd,OAAO;gBACL,GAAG,IAAI,GAAG,CAAC;oBACT,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAC3C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAC1B;oBACD,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;iBAC3C,CAAC;aACH,CAAC;IACN,CAAC;AACH,CAAC;AAED,KAAK,UAAU,2BAA2B,CACxC,OAA0B,EAC1B,OAAwB;IAExB,IAAI,CAAC;QACH,OAAO,MAAM,gBAAgB,CAC3B,OAAO,CAAC,oBAAoB,EAAE,SAAS,IAAI,EAAE,EAC7C,uBAAuB,CAAC,OAAO,CAAC,MAAM,CAAC,EACvC,OAAO,CAAC,kBAAkB,CAC3B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,4BAA4B,CACpC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,+BAA+B,EACxE,EAAE,KAAK,EAAE,CACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAC/B,SAA2B,EAC3B,MAAqB;IAErB,IACE,MAAM,CAAC,IAAI,KAAK,QAAQ;QACxB,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC1B,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,CACtC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,MAAM,CACxC,CACF;QAED,MAAM,IAAI,4BAA4B,CACpC,yDAAyD,CAC1D,CAAC;AACN,CAAC;AAED,SAAS,0BAA0B,CACjC,SAA2B,EAC3B,MAAqB;IAErB,IACE,MAAM,CAAC,IAAI,KAAK,QAAQ;QACxB,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC;QAEjD,MAAM,IAAI,4BAA4B,CACpC,0DAA0D,CAC3D,CAAC;AACN,CAAC;AAED,SAAS,0BAA0B,CACjC,SAA2B,EAC3B,YAAwC;IAExC,IACE,YAAY,CAAC,MAAM;QACnB,SAAS,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,KAAK,yBAAyB,CAAC;QAE/D,MAAM,IAAI,4BAA4B,CACpC,2DAA2D,CAC5D,CAAC;AACN,CAAC;AAED,KAAK,UAAU,0BAA0B,CACvC,OAA0B,EAC1B,KAAsB,EACtB,MAA2B;IAE3B,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACtE,uBAAuB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAC/C,oBAAoB,CAAC,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,wBAAwB,GAAG,4BAA4B,CAC3D,OAAO,EACP,KAAK,CAAC,oBAAoB,CAC3B,CAAC;IACF,MAAM,eAAe,GAAG,MAAM,2BAA2B,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC1E,wBAAwB,CAAC,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,0BAA0B,CAAC,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1D,0BAA0B,CACxB,eAAe,EACf,MAAM,CAAC,MAAM,CAAC,oBAAoB,CACnC,CAAC;IACF,OAAO,EAAE,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,CAAC;AACrE,CAAC;AAmCD,SAAS,wBAAwB,CAC/B,OAA0B,EAC1B,WAAoD;IAEpD,OAAO,OAAO,CAAC,SAAS;QACtB,CAAC,CAAC;YACE,GAAG,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,aAAa;YAC/C,GAAG,WAAW,EAAE,aAAa;SAC9B;QACH,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED,SAAS,eAAe,CAAC,IAAiB,EAAE,IAAyB;IACnE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AACzE,CAAC;AAED,SAAS,uBAAuB,CAAC,IAA6B;IAC5D,IAAI,CAAC,IAAI;QACP,MAAM,IAAI,4BAA4B,CACpC,mDAAmD,CACpD,CAAC;IACJ,OAAO,eAAe,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,gBAAgB,CACvB,OAA0B,EAC1B,QAAiB,EACjB,SAA6C;IAE7C,OAAO;QACL,SAAS;QACT,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,OAAO,CAAC,YAAY;YACtB,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YACrD,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED,SAAS,yBAAyB,CAChC,OAA0B,EAC1B,QAAiB;IAEjB,MAAM,UAAU,GAAG;QACjB,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE;QACzC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,EAAE;QACvE,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,YAAY,EAAE;KACjD,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CACtC,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,EAAE;QACpC,IAAI,IAAI,EAAE,aAAa,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QACjD,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC;IACtC,CAAC,CAAC,CACH,CAAC;IACF,OAAO,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM;QACtC,CAAC,CAAC,EAAE,iBAAiB,EAAE,aAAa,EAAE;QACtC,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED,SAAS,8BAA8B,CAAC,OAA0B;IAChE,OAAO;QACL,4BAA4B,EAC1B,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,mBAAmB,IAAI,IAAI;QACjE,GAAG,CAAC,OAAO,CAAC,SAAS;YACnB,CAAC,CAAC;gBACE,sBAAsB,EACpB,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,qBAAqB;aAClD;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED,SAAS,gCAAgC,CAAC,KAA+B;IACvE,OAAO;QACL,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY;YAC5B,CAAC,CAAC,EAAE,kBAAkB,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE;YAC9D,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,gBAAgB;YACnC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,gBAAgB,EAAE;YACzD,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,gBAAgB;YACnC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,gBAAgB,EAAE;YACzD,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED,SAAS,6BAA6B,CAAC,OAA0B;IAC/D,OAAO,OAAO,CAAC,YAAY;QACzB,CAAC,CAAC;YACE,qBAAqB,EAAE;gBACrB,GAAG,CAAC,OAAO,CAAC,qBAAqB,IAAI,EAAE,CAAC;aACzC,CAAC,IAAI,EAAE;SACT;QACH,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED,SAAS,+BAA+B,CACtC,OAA0B,EAC1B,KAA+B;IAE/B,OAAO;QACL,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU;QAC/C,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,GAAG,CAAC,OAAO,CAAC,kBAAkB;YAC5B,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,kBAAkB,EAAE;YAC7C,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,yBAAyB,CAC1B,OAAO,EACP,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CACzC;QACD,GAAG,8BAA8B,CAAC,OAAO,CAAC;QAC1C,GAAG,gCAAgC,CAAC,KAAK,CAAC;QAC1C,GAAG,KAAK,CAAC,UAAU,CAAC,aAAa;QACjC,GAAG,6BAA6B,CAAC,OAAO,CAAC;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAC9B,OAA0B,EAC1B,KAAqB;IAErB,MAAM,OAAO,GAAmB;QAC9B,GAAG,IAAI,GAAG,CACR;YACE,GAAG,KAAK,CAAC,mBAAmB;YAC5B,GAAG,KAAK,CAAC,iBAAiB;YAC1B,GAAG,KAAK,CAAC,mBAAmB;YAC5B,GAAG,KAAK,CAAC,oBAAoB;SAC9B,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAC/B;KACF;SACE,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;IAC9D,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,KAAK,MAAM,EAAE,IAAI;YACf,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;SACrE,CAAC,IAAI,EAAE;YACN,OAAO,CAAC,IAAI,CAAC;gBACX,EAAE;gBACF,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO;aACpD,CAAC,CAAC;IACP,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,2BAA2B,CAAC,OAA0B;IAC7D,OAAO;QACL,eAAe,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,YAAY,IAAI,IAAI;QACzE,iBAAiB,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI;KACxE,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CACvB,QAAsB,EACtB,aAAsC;IAEtC,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,CAAC,QAAQ,CAAC,cAAc,KAAK,SAAS;YACvC,CAAC,CAAC,EAAE,cAAc,EAAE,QAAQ,CAAC,cAAc,EAAE;YAC7C,CAAC,CAAC,EAAE,CAAC;QACP,aAAa;KACd,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAqB;IAChD,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;QACpB,KAAK,WAAW;YACd,OAAO,MAAM,CAAC,OAAO,CAAC;QACxB,KAAK,YAAY;YACf,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;QACrE,KAAK,QAAQ;YACX,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;IACnC,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,SAA2B;IACvD,OAAO,SAAS,CAAC,GAAG,CAClB,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;QACzD,EAAE;QACF,YAAY;QACZ,MAAM;QACN,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5C,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,KAA+B;IAC1D,OAAO;QACL,GAAG,mBAAmB,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAC5C,SAAS,EAAE,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC;QAC9D,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY;YAC5B,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE;YAC9C,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,gBAAgB;YACnC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,gBAAgB,EAAE;YACzD,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,gBAAgB;YACnC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,UAAU,CAAC,gBAAgB,EAAE;YACzD,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,KAAK,CAAC,UAAU,CAAC,aAAa;KAClC,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAC5B,OAA0B,EAC1B,aAAsC;IAEtC,OAAO;QACL,MAAM,EAAE,yBAAyB;QACjC,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;QACtD,aAAa;KACd,CAAC;AACJ,CAAC;AAUD,SAAS,4BAA4B,CACnC,OAA0B,EAC1B,QAAkC,EAClC,OAAyB;IAEzB,IAAI,CAAC;QACH,OAAO,wBAAwB,CAAC;YAC9B,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;YAC5C,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,mBAAmB,EAAE,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC;YACrD,GAAG,2BAA2B,CAAC,OAAO,CAAC;YACvC,UAAU,EAAE,QAAQ,CAClB,gBAAgB,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,CACtD;YACD,aAAa,EAAE,QAAQ,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YACtD,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC;YAC3C,sBAAsB,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC;YAC/D,eAAe,EAAE,QAAQ,CACvB,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,aAAa,CAAC,CACtD;YACD,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC;YAC7C,qBAAqB,EAAE,QAAQ,CAAC;gBAC9B,SAAS,EAAE,QAAQ,CAAC,wBAAwB;gBAC5C,OAAO,EAAE,QAAQ,CAAC,wBAAwB;aAC3C,CAAC;YACF,WAAW,EAAE,QAAQ,CACnB,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAC3D;YACD,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;SACrC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,4BAA4B,CACpC,oCAAoC,EACpC,EAAE,KAAK,EAAE,CACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAChC,OAA0B,EAC1B,KAA+B,EAC/B,WAAoD;IAEpD,MAAM,aAAa,GAAG,wBAAwB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,gBAAgB,CAC7B,OAAO,EACP,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EACxC,KAAK,CACN,CAAC;IACF,MAAM,cAAc,GAAG,+BAA+B,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACvE,MAAM,aAAa,GAAG,uBAAuB,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,OAAO,GAAG;QACd,aAAa;QACb,KAAK;QACL,MAAM;QACN,cAAc;QACd,aAAa;KACd,CAAC;IACF,MAAM,kBAAkB,GAAG,4BAA4B,CACrD,OAAO,EACP,KAAK,EACL,OAAO,CACR,CAAC;IACF,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,OAAO,EAAE,kBAAkB,EAAE,CAAC;AACtD,CAAC;AAED,SAAS,+BAA+B,CACtC,OAA0B,EAC1B,KAA+B;IAE/B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CACf,aAAa,CAAC,+BAA+B,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CACT,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,4BAA4B,CACpC,oCAAoC,EACpC,EAAE,KAAK,EAAE,CACV,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,OAA0B;IAE1B,MAAM,MAAM,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,MAAM,4BAA4B,CACpD,OAAO,EACP,MAAM,CAAC,MAAM,CACd,CAAC;IACF,MAAM,SAAS,GAAG,MAAM,0BAA0B,CAChD,OAAO,EACP,WAAW,EACX,MAAM,CACP,CAAC;IACF,MAAM,UAAU,GAAG,qBAAqB,CAAC,OAAO,EAAE;QAChD,oBAAoB,EAAE,WAAW,CAAC,oBAAoB;QACtD,eAAe,EAAE,SAAS,CAAC,eAAe;KAC3C,CAAC,CAAC;IACH,MAAM,QAAQ,GAA6B;QACzC,OAAO,EAAE;YACP,WAAW,EAAE,WAAW,CAAC,WAAW;YACpC,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,YAAY,EAAE,SAAS,CAAC,YAAY;YACpC,eAAe,EAAE,SAAS,CAAC,eAAe;SAC3C;QACD,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,UAAU;QACV,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,wBAAwB,EAAE,SAAS,CAAC,wBAAwB;QAC5D,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,OAAO,EAAE,WAAW,CAAC,OAAO;KAC7B,CAAC;IACF,OAAO,yBAAyB,CAC9B,OAAO,EACP,QAAQ,EACR,WAAW,CAAC,oBAAoB,CACjC,CAAC;AACJ,CAAC","sourcesContent":["import { realpath } from \"node:fs/promises\";\nimport {\n  EvaluationConfigurationError,\n  type EvaluationOptions,\n  type HostAdapter,\n  type ResolvedCase,\n} from \"./types\";\nimport { prepareFixtureSetup, type FixtureSetup } from \"../fixtures/setup\";\nimport type { FixtureSource } from \"../fixtures/create\";\nimport { type SemanticCheckDeclaration } from \"../graders/semantic\";\nimport { canonicalJson, createEvaluationIdentity, hashJson } from \"../identity\";\nimport {\n  prepareInstrumentation,\n  type InstrumentationRequest,\n} from \"../instrumentation\";\nimport {\n  fixtureParts,\n  prepareArtifacts,\n  type InlineArtifact,\n} from \"../fixtures/preparation\";\nimport { projectProvenance, runnerProvenance } from \"../provenance\";\nimport { resolveRepositorySource } from \"../fixtures/repository\";\nimport {\n  prepareEvaluationChecks,\n  type EvaluationCheckPlan,\n  type PreparedChecks,\n} from \"./check-plan\";\nimport {\n  prepareInvocationPlan,\n  type PreparedInvocation,\n} from \"./invocation-plan\";\nimport type { PreparationResult } from \"../extension-session\";\n\nexport type { PreparedChecks } from \"./check-plan\";\nexport type { PreparedInvocation } from \"./invocation-plan\";\n\ninterface EnvironmentPlan {\n  readonly projectRoot: string;\n  readonly source: FixtureSource;\n  readonly runner: Awaited<ReturnType<typeof runnerProvenance>>;\n  readonly project: Awaited<ReturnType<typeof projectProvenance>>;\n  readonly extensionPreparation: PreparationResult | null;\n}\n\ninterface ArtifactPlan {\n  readonly fixtureSetup: FixtureSetup | null;\n  readonly requestedInstrumentation: InstrumentationRequest[];\n  readonly inlineArtifacts: InlineArtifact[];\n}\n\nasync function prepareEvaluationEnvironment(\n  options: EvaluationOptions,\n  source: FixtureSource<null>,\n): Promise<EnvironmentPlan> {\n  const projectRoot = await realpath(options.projectRoot).catch(() => {\n    throw new EvaluationConfigurationError(\"project root is unreadable\");\n  });\n  const resolvedSource = await resolveFixtureSource(options, source);\n  const [runner, project] = await Promise.all([\n    runnerProvenance(\n      options.runnerBuildDigest,\n      options.runnerCheckoutRoot,\n    ).catch((error: unknown) => {\n      throw new EvaluationConfigurationError(\n        error instanceof Error ? error.message : \"invalid runner provenance\",\n      );\n    }),\n    projectProvenance(projectRoot, evaluationStorageRoots(options)),\n  ]);\n  const extensionPreparation = options.extension\n    ? await options.extension.session.prepare(\n        options.extension.resolvedCase,\n        {\n          id: options.host.id,\n          capabilities: [\n            ...(options.host.instrumentation ?? []).map((item) => item.id),\n            ...(options.host.hostCapabilities ?? []),\n          ],\n        },\n        options.condition,\n      )\n    : null;\n  return {\n    projectRoot,\n    source: resolvedSource,\n    runner,\n    project,\n    extensionPreparation,\n  };\n}\n\nasync function resolveFixtureSource(\n  options: EvaluationOptions,\n  source: FixtureSource<null>,\n): Promise<FixtureSource> {\n  if (source.kind !== \"repository\") return source;\n  const repository = await resolveRepositorySource(\n    source.fixture.sourceRef,\n    options.preparationSources,\n  ).catch((error: unknown) => {\n    throw new EvaluationConfigurationError(\n      error instanceof Error ? error.message : \"invalid repository source\",\n    );\n  });\n  return { ...source, repository };\n}\n\nfunction evaluationStorageRoots(options: EvaluationOptions): string[] {\n  return [options.resultsRoot, options.runStateRoot ?? options.resultsRoot];\n}\n\nfunction preparedFixtureSetup(\n  preparation: EnvironmentPlan[\"extensionPreparation\"],\n): FixtureSetup | null {\n  try {\n    return prepareFixtureSetup(preparation?.fixtureSetup);\n  } catch (cause) {\n    throw new EvaluationConfigurationError(\n      cause instanceof Error ? cause.message : \"invalid fixture setup\",\n      { cause },\n    );\n  }\n}\n\nfunction validateSetupCapability(\n  options: EvaluationOptions,\n  setup: FixtureSetup | null,\n): void {\n  if (\n    setup &&\n    !options.extension?.session.identity.capabilities.includes(\n      \"sevro.fixture.setup\",\n    )\n  )\n    throw new EvaluationConfigurationError(\n      \"fixture setup capability was not negotiated\",\n    );\n}\n\nfunction validateSetupFixture(\n  setup: FixtureSetup | null,\n  source: FixtureSource,\n): void {\n  if (setup && source.kind === \"inline\")\n    throw new EvaluationConfigurationError(\n      \"fixture setup requires a Git fixture\",\n    );\n}\n\nfunction negotiatedInstrumentationCapabilities(\n  options: EvaluationOptions,\n): string[] {\n  return options.extension?.session.identity.capabilities ?? [];\n}\n\nfunction configurationFailureMessage(cause: unknown, fallback: string): string {\n  return cause instanceof Error ? cause.message : fallback;\n}\n\nfunction requestedInstrumentationPlan(\n  options: EvaluationOptions,\n  preparation: EnvironmentPlan[\"extensionPreparation\"],\n): InstrumentationRequest[] {\n  try {\n    return prepareInstrumentation(\n      preparation?.requestedInstrumentation ?? [],\n      options.host.instrumentation ?? [],\n      negotiatedInstrumentationCapabilities(options),\n      options.condition,\n    );\n  } catch (cause) {\n    throw new EvaluationConfigurationError(\n      configurationFailureMessage(cause, \"invalid instrumentation\"),\n      { cause },\n    );\n  }\n}\n\nfunction fixturePreparationPaths(source: FixtureSource): string[] {\n  switch (source.kind) {\n    case \"inline\":\n      return Object.keys(source.files ?? {});\n    case \"repository\":\n      return Object.keys(source.fixture.files ?? {});\n    case \"generated\":\n      return [\n        ...new Set([\n          ...source.fixture.commits.flatMap((commit) =>\n            Object.keys(commit.files),\n          ),\n          ...Object.keys(source.fixture.files ?? {}),\n        ]),\n      ];\n  }\n}\n\nasync function preparedEvaluationArtifacts(\n  options: EvaluationOptions,\n  context: EnvironmentPlan,\n): Promise<InlineArtifact[]> {\n  try {\n    return await prepareArtifacts(\n      context.extensionPreparation?.artifacts ?? [],\n      fixturePreparationPaths(context.source),\n      options.preparationSources,\n    );\n  } catch (cause) {\n    throw new EvaluationConfigurationError(\n      cause instanceof Error ? cause.message : \"invalid preparation artifacts\",\n      { cause },\n    );\n  }\n}\n\nfunction validateArtifactMetadata(\n  artifacts: InlineArtifact[],\n  source: FixtureSource,\n): void {\n  if (\n    source.kind !== \"inline\" &&\n    artifacts.some((artifact) =>\n      fixtureParts(artifact.relativePath).some(\n        (part) => part.toLowerCase() === \".git\",\n      ),\n    )\n  )\n    throw new EvaluationConfigurationError(\n      \"preparation artifacts cannot modify repository metadata\",\n    );\n}\n\nfunction validateArtifactExclusions(\n  artifacts: InlineArtifact[],\n  source: FixtureSource,\n): void {\n  if (\n    source.kind === \"inline\" &&\n    artifacts.some((artifact) => artifact.gitExclude)\n  )\n    throw new EvaluationConfigurationError(\n      \"Git-excluded preparation artifacts require a Git fixture\",\n    );\n}\n\nfunction validateSemanticArtifactId(\n  artifacts: InlineArtifact[],\n  declarations: SemanticCheckDeclaration[],\n): void {\n  if (\n    declarations.length &&\n    artifacts.some((item) => item.id === \"sevro.semantic.verdicts\")\n  )\n    throw new EvaluationConfigurationError(\n      \"preparation artifact uses a reserved semantic evidence ID\",\n    );\n}\n\nasync function prepareEvaluationArtifacts(\n  options: EvaluationOptions,\n  state: EnvironmentPlan,\n  checks: EvaluationCheckPlan,\n): Promise<ArtifactPlan> {\n  const fixtureSetup = preparedFixtureSetup(state.extensionPreparation);\n  validateSetupCapability(options, fixtureSetup);\n  validateSetupFixture(fixtureSetup, state.source);\n  const requestedInstrumentation = requestedInstrumentationPlan(\n    options,\n    state.extensionPreparation,\n  );\n  const inlineArtifacts = await preparedEvaluationArtifacts(options, state);\n  validateArtifactMetadata(inlineArtifacts, state.source);\n  validateArtifactExclusions(inlineArtifacts, state.source);\n  validateSemanticArtifactId(\n    inlineArtifacts,\n    checks.checks.semanticDeclarations,\n  );\n  return { fixtureSetup, requestedInstrumentation, inlineArtifacts };\n}\n\ntype EvaluationRouteRole = \"candidate\" | \"semantic\" | \"advisory\";\ninterface ActiveGrader {\n  id: string;\n  source: \"builtin\" | \"extension\";\n  version: string;\n}\n\nexport interface PreparedFixture {\n  readonly projectRoot: string;\n  readonly source: FixtureSource;\n  readonly fixtureSetup: ArtifactPlan[\"fixtureSetup\"];\n  readonly inlineArtifacts: InlineArtifact[];\n}\n\ninterface PreparedEvaluationInputs {\n  readonly fixture: PreparedFixture;\n  readonly checks: PreparedChecks;\n  readonly invocation: PreparedInvocation;\n  readonly jobs: number;\n  readonly requestedInstrumentation: InstrumentationRequest[];\n  readonly runner: EnvironmentPlan[\"runner\"];\n  readonly project: EnvironmentPlan[\"project\"];\n}\n\nexport interface PreparedEvaluation extends PreparedEvaluationInputs {\n  readonly extensionData: Record<string, unknown>;\n  readonly route: ReturnType<typeof evaluationRoute>;\n  readonly routes: ReturnType<typeof evaluationRoutes>;\n  readonly redactedConfig: ReturnType<typeof redactedEvaluationConfiguration>;\n  readonly activeGraders: ActiveGrader[];\n  readonly evaluationIdentity: ReturnType<typeof createEvaluationIdentity>;\n}\n\nfunction extensionPreparationData(\n  options: EvaluationOptions,\n  preparation: EnvironmentPlan[\"extensionPreparation\"],\n): Record<string, unknown> {\n  return options.extension\n    ? {\n        ...options.extension.resolvedCase.extensionData,\n        ...preparation?.extensionData,\n      }\n    : {};\n}\n\nfunction evaluationRoute(host: HostAdapter, role: EvaluationRouteRole) {\n  return { role, host: host.id, model: host.model, effort: host.effort };\n}\n\nfunction semanticEvaluationRoute(host: HostAdapter | undefined) {\n  if (!host)\n    throw new EvaluationConfigurationError(\n      \"semantic checks require an explicit semantic host\",\n    );\n  return evaluationRoute(host, \"semantic\");\n}\n\nfunction evaluationRoutes(\n  options: EvaluationOptions,\n  semantic: boolean,\n  candidate: ReturnType<typeof evaluationRoute>,\n) {\n  return [\n    candidate,\n    ...(semantic ? [semanticEvaluationRoute(options.semanticHost)] : []),\n    ...(options.advisoryHost\n      ? [evaluationRoute(options.advisoryHost, \"advisory\")]\n      : []),\n  ];\n}\n\nfunction redactedHostConfiguration(\n  options: EvaluationOptions,\n  semantic: boolean,\n) {\n  const selections = [\n    { role: \"candidate\", host: options.host },\n    { role: \"semantic\", host: semantic ? options.semanticHost : undefined },\n    { role: \"advisory\", host: options.advisoryHost },\n  ];\n  const configuration = Object.fromEntries(\n    selections.flatMap(({ role, host }) => {\n      if (host?.configuration === undefined) return [];\n      return [[role, host.configuration]];\n    }),\n  );\n  return Object.keys(configuration).length\n    ? { hostConfiguration: configuration }\n    : {};\n}\n\nfunction redactedExtensionConfiguration(options: EvaluationOptions) {\n  return {\n    extensionConfigurationDigest:\n      options.extension?.session.identity.configurationDigest ?? null,\n    ...(options.extension\n      ? {\n          extensionConfiguration:\n            options.extension.session.redactedConfiguration,\n        }\n      : {}),\n  };\n}\n\nfunction redactedPreparationConfiguration(state: PreparedEvaluationInputs) {\n  return {\n    ...(state.fixture.fixtureSetup\n      ? { fixtureSetupDigest: hashJson(state.fixture.fixtureSetup) }\n      : {}),\n    ...(state.invocation.codexMarketplace\n      ? { codexMarketplace: state.invocation.codexMarketplace }\n      : {}),\n    ...(state.invocation.claudePluginDirs\n      ? { claudePluginDirs: state.invocation.claudePluginDirs }\n      : {}),\n  };\n}\n\nfunction redactedAdvisoryConfiguration(options: EvaluationOptions) {\n  return options.advisoryHost\n    ? {\n        advisoryExcludedPaths: [\n          ...(options.advisoryExcludedPaths ?? []),\n        ].sort(),\n      }\n    : {};\n}\n\nfunction redactedEvaluationConfiguration(\n  options: EvaluationOptions,\n  state: PreparedEvaluationInputs,\n) {\n  return {\n    condition: options.condition,\n    executionMode: options.dry ? \"dry\" : \"executed\",\n    ...(options.runtimePolicy ? { runtimePolicy: options.runtimePolicy } : {}),\n    trialCount: options.trialCount,\n    jobs: state.jobs,\n    passThreshold: options.passThreshold,\n    ...(options.assessmentSettings\n      ? { runSettings: options.assessmentSettings }\n      : {}),\n    ...redactedHostConfiguration(\n      options,\n      state.checks.preparedSemantic.length > 0,\n    ),\n    ...redactedExtensionConfiguration(options),\n    ...redactedPreparationConfiguration(state),\n    ...state.invocation.configuration,\n    ...redactedAdvisoryConfiguration(options),\n  };\n}\n\nfunction activeEvaluationGraders(\n  options: EvaluationOptions,\n  state: PreparedChecks,\n): ActiveGrader[] {\n  const graders: ActiveGrader[] = [\n    ...new Set(\n      [\n        ...state.builtinDeclarations,\n        ...state.shellDeclarations,\n        ...state.gitHeadDeclarations,\n        ...state.semanticDeclarations,\n      ].map((check) => check.grader),\n    ),\n  ]\n    .sort()\n    .map((id) => ({ id, source: \"builtin\", version: \"1.0.0\" }));\n  if (options.extension) {\n    for (const id of [\n      ...new Set(state.extensionDeclarations.map((check) => check.grader)),\n    ].sort())\n      graders.push({\n        id,\n        source: \"extension\",\n        version: options.extension.session.identity.version,\n      });\n  }\n  return graders;\n}\n\nfunction extensionIdentityDimensions(options: EvaluationOptions) {\n  return {\n    extensionDigest: options.extension?.session.identity.sourceDigest ?? null,\n    extensionProtocol: options.extension?.session.identity.protocol ?? null,\n  };\n}\n\nfunction caseIdentityData(\n  caseData: ResolvedCase,\n  extensionData: Record<string, unknown>,\n) {\n  return {\n    id: caseData.id,\n    prompt: caseData.prompt,\n    ...(caseData.followUpPrompt !== undefined\n      ? { followUpPrompt: caseData.followUpPrompt }\n      : {}),\n    extensionData,\n  };\n}\n\nfunction baseFixtureIdentity(source: FixtureSource) {\n  switch (source.kind) {\n    case \"generated\":\n      return source.fixture;\n    case \"repository\":\n      return { ...source.fixture, revision: source.repository.revision };\n    case \"inline\":\n      return { files: source.files };\n  }\n}\n\nfunction artifactIdentityData(artifacts: InlineArtifact[]) {\n  return artifacts.map(\n    ({ id, relativePath, sha256, gitExclude, executable }) => ({\n      id,\n      relativePath,\n      sha256,\n      ...(gitExclude ? { gitExclude: true } : {}),\n      ...(executable ? { executable: true } : {}),\n    }),\n  );\n}\n\nfunction fixtureIdentityData(state: PreparedEvaluationInputs) {\n  return {\n    ...baseFixtureIdentity(state.fixture.source),\n    artifacts: artifactIdentityData(state.fixture.inlineArtifacts),\n    ...(state.fixture.fixtureSetup\n      ? { fixtureSetup: state.fixture.fixtureSetup }\n      : {}),\n    ...(state.invocation.codexMarketplace\n      ? { codexMarketplace: state.invocation.codexMarketplace }\n      : {}),\n    ...(state.invocation.claudePluginDirs\n      ? { claudePluginDirs: state.invocation.claudePluginDirs }\n      : {}),\n    ...state.invocation.configuration,\n  };\n}\n\nfunction evaluatorIdentityData(\n  options: EvaluationOptions,\n  extensionData: Record<string, unknown>,\n) {\n  return {\n    policy: \"sevro.builtin-output.v1\",\n    extension: options.extension?.session.identity ?? null,\n    extensionData,\n  };\n}\n\ntype IdentityMetadata = {\n  extensionData: Record<string, unknown>;\n  route: ReturnType<typeof evaluationRoute>;\n  routes: ReturnType<typeof evaluationRoutes>;\n  redactedConfig: ReturnType<typeof redactedEvaluationConfiguration>;\n  activeGraders: ActiveGrader[];\n};\n\nfunction evaluationComparisonIdentity(\n  options: EvaluationOptions,\n  prepared: PreparedEvaluationInputs,\n  context: IdentityMetadata,\n) {\n  try {\n    return createEvaluationIdentity({\n      runnerBuildDigest: options.runnerBuildDigest,\n      projectDigest: options.projectDigest,\n      configurationDigest: hashJson(context.redactedConfig),\n      ...extensionIdentityDimensions(options),\n      caseDigest: hashJson(\n        caseIdentityData(options.case, context.extensionData),\n      ),\n      fixtureDigest: hashJson(fixtureIdentityData(prepared)),\n      checksDigest: hashJson(options.case.checks),\n      requiredEvidenceDigest: hashJson(options.case.requiredEvidence),\n      evaluatorDigest: hashJson(\n        evaluatorIdentityData(options, context.extensionData),\n      ),\n      graderDigest: hashJson(context.activeGraders),\n      instrumentationDigest: hashJson({\n        requested: prepared.requestedInstrumentation,\n        applied: prepared.requestedInstrumentation,\n      }),\n      routeDigest: hashJson(\n        context.routes.length > 1 ? context.routes : context.route,\n      ),\n      condition: options.condition,\n      trialCount: options.trialCount,\n      passThreshold: options.passThreshold,\n    });\n  } catch (cause) {\n    throw new EvaluationConfigurationError(\n      \"invalid evaluation identity inputs\",\n      { cause },\n    );\n  }\n}\n\nfunction prepareEvaluationIdentity(\n  options: EvaluationOptions,\n  state: PreparedEvaluationInputs,\n  preparation: EnvironmentPlan[\"extensionPreparation\"],\n): PreparedEvaluation {\n  const extensionData = extensionPreparationData(options, preparation);\n  const route = evaluationRoute(options.host, \"candidate\");\n  const routes = evaluationRoutes(\n    options,\n    state.checks.preparedSemantic.length > 0,\n    route,\n  );\n  const redactedConfig = snapshotEvaluationConfiguration(options, state);\n  const activeGraders = activeEvaluationGraders(options, state.checks);\n  const context = {\n    extensionData,\n    route,\n    routes,\n    redactedConfig,\n    activeGraders,\n  };\n  const evaluationIdentity = evaluationComparisonIdentity(\n    options,\n    state,\n    context,\n  );\n  return { ...state, ...context, evaluationIdentity };\n}\n\nfunction snapshotEvaluationConfiguration(\n  options: EvaluationOptions,\n  state: PreparedEvaluationInputs,\n): ReturnType<typeof redactedEvaluationConfiguration> {\n  try {\n    return JSON.parse(\n      canonicalJson(redactedEvaluationConfiguration(options, state)),\n    ) as ReturnType<typeof redactedEvaluationConfiguration>;\n  } catch (cause) {\n    throw new EvaluationConfigurationError(\n      \"invalid evaluation identity inputs\",\n      { cause },\n    );\n  }\n}\n\nexport async function prepareEvaluation(\n  options: EvaluationOptions,\n): Promise<PreparedEvaluation> {\n  const checks = prepareEvaluationChecks(options);\n  const environment = await prepareEvaluationEnvironment(\n    options,\n    checks.source,\n  );\n  const artifacts = await prepareEvaluationArtifacts(\n    options,\n    environment,\n    checks,\n  );\n  const invocation = prepareInvocationPlan(options, {\n    extensionPreparation: environment.extensionPreparation,\n    inlineArtifacts: artifacts.inlineArtifacts,\n  });\n  const prepared: PreparedEvaluationInputs = {\n    fixture: {\n      projectRoot: environment.projectRoot,\n      source: environment.source,\n      fixtureSetup: artifacts.fixtureSetup,\n      inlineArtifacts: artifacts.inlineArtifacts,\n    },\n    checks: checks.checks,\n    invocation,\n    jobs: checks.jobs,\n    requestedInstrumentation: artifacts.requestedInstrumentation,\n    runner: environment.runner,\n    project: environment.project,\n  };\n  return prepareEvaluationIdentity(\n    options,\n    prepared,\n    environment.extensionPreparation,\n  );\n}\n"]}

package/dist/src/evaluation/progress.js

@@ -0,0 +1,14 @@
+/** Observation is best-effort and cannot change assessment or persistence. */
+export function evaluationProgress(options, stage, status, trial) {
+    try {
+        options.onProgress?.({
+            stage,
+            status,
+            ...(trial === undefined ? {} : { trial }),
+        });
+    }
+    catch {
+        // An observer has no authority over the evaluation lifecycle.
+    }
+}
+//# sourceMappingURL=progress.js.map

package/dist/src/evaluation/progress.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.js","sourceRoot":"","sources":["progress.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,MAAM,UAAU,kBAAkB,CAChC,OAA0B,EAC1B,KAAkC,EAClC,MAAoC,EACpC,KAAc;IAEd,IAAI,CAAC;QACH,OAAO,CAAC,UAAU,EAAE,CAAC;YACnB,KAAK;YACL,MAAM;YACN,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;SAC1C,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;IAChE,CAAC;AACH,CAAC","sourcesContent":["import type { EvaluationOptions, EvaluationProgress } from \"./types\";\n\n/** Observation is best-effort and cannot change assessment or persistence. */\nexport function evaluationProgress(\n  options: EvaluationOptions,\n  stage: EvaluationProgress[\"stage\"],\n  status: EvaluationProgress[\"status\"],\n  trial?: number,\n): void {\n  try {\n    options.onProgress?.({\n      stage,\n      status,\n      ...(trial === undefined ? {} : { trial }),\n    });\n  } catch {\n    // An observer has no authority over the evaluation lifecycle.\n  }\n}\n"]}

package/dist/src/evaluation/resources.js

@@ -0,0 +1,159 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, realpath, rm, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import { allocateWorkspace, retainWorkspace, removeWorkspace, } from "../fixtures/workspace-state.js";
+import { createFixture } from "../fixtures/create.js";
+import { fixtureBinDirectory } from "../fixtures/bin.js";
+import { fixtureParts } from "../fixtures/preparation.js";
+import { startRunOwner } from "../run-owner.js";
+import { EvaluationConfigurationError } from "./types.js";
+import { INVOCATION_PLACEHOLDER, LEGACY_CODEX_PLACEHOLDER, } from "./invocation-plan.js";
+/** Reservations outlive completed contents until all admitted trials drain. */
+export class RunWorkspaceReservations {
+    candidates = [];
+    semantics = [];
+    retained = new Set();
+    async reserve(input) {
+        for (let trial = 1; trial <= input.count && !input.signal?.aborted; trial++) {
+            this.candidates.push(await allocateWorkspace());
+            if (input.semantic)
+                this.semantics.push(await allocateWorkspace());
+        }
+    }
+    candidate(trial) {
+        const reservation = this.candidates[trial - 1];
+        if (reservation === undefined)
+            throw new Error(`trial ${trial} workspace was not reserved`);
+        return reservation;
+    }
+    semantic(trial) {
+        return this.semantics[trial - 1];
+    }
+    retain(workspace) {
+        this.retained.add(workspace);
+    }
+    completed(reservation) {
+        this.retained.delete(reservation);
+    }
+    async cleanup() {
+        for (const workspace of this.retained)
+            await retainWorkspace(workspace);
+        await Promise.all([...this.candidates, ...this.semantics]
+            .filter((workspace) => !this.retained.has(workspace))
+            .map(async (workspace) => {
+            try {
+                await removeWorkspace(workspace);
+            }
+            catch {
+                console.warn(`fixture cleanup failed; retained at ${workspace}`);
+            }
+        }));
+    }
+}
+export async function allocateRunResources(options, evaluationDigest) {
+    const runId = randomUUID();
+    await mkdir(options.resultsRoot, { recursive: true, mode: 0o700 });
+    const runDir = resolve(await realpath(options.resultsRoot), runId);
+    await mkdir(runDir, { mode: 0o700 });
+    const runStateRoot = options.runStateRoot ?? options.resultsRoot;
+    await mkdir(runStateRoot, { recursive: true, mode: 0o700 });
+    const stateRoot = await realpath(runStateRoot);
+    const stateDir = join(stateRoot, runId);
+    if (stateDir !== runDir)
+        await mkdir(stateDir, { mode: 0o700 });
+    const activePath = join(stateRoot, "active", `${runId}.json`);
+    const checkpointPath = join(stateDir, "checkpoint.json");
+    const evidencePath = join(runDir, "run.json");
+    let owner;
+    try {
+        owner = startRunOwner({
+            stateRoot,
+            evaluationDigest,
+            attemptId: runId,
+            activeRunPath: activePath,
+            checkpointPath,
+            evidenceDirectory: stateDir,
+            artifactPath: evidencePath,
+        });
+    }
+    catch (error) {
+        await rm(runDir, { recursive: true, force: true });
+        if (stateDir !== runDir)
+            await rm(stateDir, { recursive: true, force: true });
+        throw error;
+    }
+    return {
+        runId,
+        runDir,
+        runStateRoot,
+        stateDir,
+        evidencePath,
+        owner,
+        workspaces: new RunWorkspaceReservations(),
+    };
+}
+export async function retainPreparationArtifacts(artifacts, runDir) {
+    const artifactRefs = [];
+    for (const artifact of artifacts) {
+        const retainedPath = join(runDir, "prepared", ...fixtureParts(artifact.relativePath));
+        await mkdir(dirname(retainedPath), { recursive: true, mode: 0o700 });
+        await writeFile(retainedPath, artifact.bytes, {
+            flag: "wx",
+            mode: artifact.executable ? 0o700 : 0o600,
+        });
+        artifactRefs.push({
+            id: artifact.id,
+            path: pathToFileURL(retainedPath).href,
+            sha256: artifact.sha256,
+            ...(artifact.gitExclude ? { gitExclude: true } : {}),
+            ...(artifact.executable ? { executable: true } : {}),
+        });
+    }
+    return artifactRefs;
+}
+function renderTrialPrompt(prompt, workspace, invocation) {
+    return prompt
+        .replaceAll("{{sevro.workspace}}", workspace)
+        .replaceAll(INVOCATION_PLACEHOLDER, invocation.invocationToken ?? INVOCATION_PLACEHOLDER)
+        .replaceAll(LEGACY_CODEX_PLACEHOLDER, invocation.invocationToken ?? LEGACY_CODEX_PLACEHOLDER);
+}
+export async function prepareTrialWorkspace(options, fixture, invocation, workspaces, trial, candidateRuntimeRoot) {
+    const reservation = workspaces.candidate(trial);
+    workspaces.retain(reservation);
+    const workspace = await createFixture(fixture.source, {
+        artifacts: fixture.inlineArtifacts,
+        sources: options.preparationSources,
+        setup: fixture.fixtureSetup,
+        projectRoot: fixture.projectRoot,
+        signal: options.signal,
+        reservedWorkspace: reservation,
+        candidateRuntimeRoot,
+    }).catch((error) => {
+        if (!options.signal?.aborted ||
+            error instanceof EvaluationConfigurationError)
+            throw error;
+        return null;
+    });
+    const details = await trialWorkspaceDetails(options.case, invocation, reservation, workspace);
+    return { ...details, candidateRuntimeRoot };
+}
+async function trialWorkspaceDetails(caseData, invocation, reservation, preparedWorkspace) {
+    const workspace = preparedWorkspace ?? reservation;
+    const fixtureBinDir = preparedWorkspace === null
+        ? undefined
+        : ((await fixtureBinDirectory(workspace)) ?? undefined);
+    const trialPrompt = renderTrialPrompt(caseData.prompt, workspace, invocation);
+    const trialFollowUpPrompt = caseData.followUpPrompt === undefined
+        ? undefined
+        : renderTrialPrompt(caseData.followUpPrompt, workspace, invocation);
+    return {
+        reservation,
+        workspace,
+        preparationCancelled: preparedWorkspace === null,
+        fixtureBinDir,
+        trialPrompt,
+        trialFollowUpPrompt,
+    };
+}
+//# sourceMappingURL=resources.js.map

package/dist/src/evaluation/resources.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resources.js","sourceRoot":"","sources":["resources.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,oBAAoB;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,yBAAyB;AAClE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,kBAAkB;AACnD,OAAO,EAAE,aAAa,EAAE,iBAAiB;AACzC,OAAO,EACL,iBAAiB,EACjB,eAAe,EACf,eAAe,GAChB,uCAAoC;AACrC,OAAO,EAAE,aAAa,EAAE,8BAA2B;AACnD,OAAO,EAAE,mBAAmB,EAAE,2BAAwB;AACtD,OAAO,EAAE,YAAY,EAAE,mCAAgC;AACvD,OAAO,EAAE,aAAa,EAAiB,wBAAqB;AAC5D,OAAO,EAAE,4BAA4B,EAA0B,mBAAgB;AAE/E,OAAO,EACL,sBAAsB,EACtB,wBAAwB,GACzB,6BAA0B;AAI3B,+EAA+E;AAC/E,MAAM,OAAO,wBAAwB;IAClB,UAAU,GAAa,EAAE,CAAC;IAC1B,SAAS,GAAa,EAAE,CAAC;IACzB,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAE9C,KAAK,CAAC,OAAO,CAAC,KAIb;QACC,KACE,IAAI,KAAK,GAAG,CAAC,EACb,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAC9C,KAAK,EAAE,EACP,CAAC;YACD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC;YAChD,IAAI,KAAK,CAAC,QAAQ;gBAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,SAAS,CAAC,KAAa;QACrB,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC/C,IAAI,WAAW,KAAK,SAAS;YAC3B,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,6BAA6B,CAAC,CAAC;QAC/D,OAAO,WAAW,CAAC;IACrB,CAAC;IAED,QAAQ,CAAC,KAAa;QACpB,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,CAAC,SAAiB;QACtB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;IAED,SAAS,CAAC,WAAmB;QAC3B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,OAAO;QACX,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,QAAQ;YAAE,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QACxE,MAAM,OAAO,CAAC,GAAG,CACf,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;aACpC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;aACpD,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;YACnC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,IAAI,CAAC,uCAAuC,SAAS,EAAE,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC,CACL,CAAC;IACJ,CAAC;CACF;AAYD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAgE,EAChE,gBAAwB;IAExB,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC;IAC3B,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACrC,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,WAAW,CAAC;IACjE,MAAM,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACxC,IAAI,QAAQ,KAAK,MAAM;QAAE,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAChE,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,KAAK,OAAO,CAAC,CAAC;IAC9D,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;IACzD,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC9C,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,aAAa,CAAC;YACpB,SAAS;YACT,gBAAgB;YAChB,SAAS,EAAE,KAAK;YAChB,aAAa,EAAE,UAAU;YACzB,cAAc;YACd,iBAAiB,EAAE,QAAQ;YAC3B,YAAY,EAAE,YAAY;SAC3B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,IAAI,QAAQ,KAAK,MAAM;YACrB,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,MAAM,KAAK,CAAC;IACd,CAAC;IACD,OAAO;QACL,KAAK;QACL,MAAM;QACN,YAAY;QACZ,QAAQ;QACR,YAAY;QACZ,KAAK;QACL,UAAU,EAAE,IAAI,wBAAwB,EAAE;KAC3C,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,SAA6C,EAC7C,MAAc;IAEd,MAAM,YAAY,GAAmC,EAAE,CAAC;IACxD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,YAAY,GAAG,IAAI,CACvB,MAAM,EACN,UAAU,EACV,GAAG,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,CACvC,CAAC;QACF,MAAM,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACrE,MAAM,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE;YAC5C,IAAI,EAAE,IAAI;YACV,IAAI,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK;SAC1C,CAAC,CAAC;QACH,YAAY,CAAC,IAAI,CAAC;YAChB,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI,EAAE,aAAa,CAAC,YAAY,CAAC,CAAC,IAAI;YACtC,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,SAAS,iBAAiB,CACxB,MAAc,EACd,SAAiB,EACjB,UAA8B;IAE9B,OAAO,MAAM;SACV,UAAU,CAAC,qBAAqB,EAAE,SAAS,CAAC;SAC5C,UAAU,CACT,sBAAsB,EACtB,UAAU,CAAC,eAAe,IAAI,sBAAsB,CACrD;SACA,UAAU,CACT,wBAAwB,EACxB,UAAU,CAAC,eAAe,IAAI,wBAAwB,CACvD,CAAC;AACN,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,OAAgC,EAChC,OAAwB,EACxB,UAA8B,EAC9B,UAAoC,EACpC,KAAa,EACb,oBAA6B;IAE7B,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAChD,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC/B,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE;QACpD,SAAS,EAAE,OAAO,CAAC,eAAe;QAClC,OAAO,EAAE,OAAO,CAAC,kBAAkB;QACnC,KAAK,EAAE,OAAO,CAAC,YAAY;QAC3B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,iBAAiB,EAAE,WAAW;QAC9B,oBAAoB;KACrB,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QAC1B,IACE,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO;YACxB,KAAK,YAAY,4BAA4B;YAE7C,MAAM,KAAK,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,qBAAqB,CACzC,OAAO,CAAC,IAAI,EACZ,UAAU,EACV,WAAW,EACX,SAAS,CACV,CAAC;IACF,OAAO,EAAE,GAAG,OAAO,EAAE,oBAAoB,EAAE,CAAC;AAC9C,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,QAAmC,EACnC,UAA8B,EAC9B,WAAmB,EACnB,iBAAgC;IAEhC,MAAM,SAAS,GAAG,iBAAiB,IAAI,WAAW,CAAC;IACnD,MAAM,aAAa,GACjB,iBAAiB,KAAK,IAAI;QACxB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,SAAS,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC;IAC5D,MAAM,WAAW,GAAG,iBAAiB,CAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IAC9E,MAAM,mBAAmB,GACvB,QAAQ,CAAC,cAAc,KAAK,SAAS;QACnC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,cAAc,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IACxE,OAAO;QACL,WAAW;QACX,SAAS;QACT,oBAAoB,EAAE,iBAAiB,KAAK,IAAI;QAChD,aAAa;QACb,WAAW;QACX,mBAAmB;KACpB,CAAC;AACJ,CAAC","sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { mkdir, realpath, rm, writeFile } from \"node:fs/promises\";\nimport { dirname, join, resolve } from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\nimport {\n  allocateWorkspace,\n  retainWorkspace,\n  removeWorkspace,\n} from \"../fixtures/workspace-state\";\nimport { createFixture } from \"../fixtures/create\";\nimport { fixtureBinDirectory } from \"../fixtures/bin\";\nimport { fixtureParts } from \"../fixtures/preparation\";\nimport { startRunOwner, type RunOwner } from \"../run-owner\";\nimport { EvaluationConfigurationError, type EvaluationOptions } from \"./types\";\nimport type { PreparedFixture, PreparedInvocation } from \"./plan\";\nimport {\n  INVOCATION_PLACEHOLDER,\n  LEGACY_CODEX_PLACEHOLDER,\n} from \"./invocation-plan\";\nimport type { ArtifactReference as PreparationArtifactReference } from \"../contract-types\";\nexport type { ArtifactReference as PreparationArtifactReference } from \"../contract-types\";\n\n/** Reservations outlive completed contents until all admitted trials drain. */\nexport class RunWorkspaceReservations {\n  private readonly candidates: string[] = [];\n  private readonly semantics: string[] = [];\n  private readonly retained = new Set<string>();\n\n  async reserve(input: {\n    count: number;\n    semantic: boolean;\n    signal?: AbortSignal;\n  }): Promise<void> {\n    for (\n      let trial = 1;\n      trial <= input.count && !input.signal?.aborted;\n      trial++\n    ) {\n      this.candidates.push(await allocateWorkspace());\n      if (input.semantic) this.semantics.push(await allocateWorkspace());\n    }\n  }\n\n  candidate(trial: number): string {\n    const reservation = this.candidates[trial - 1];\n    if (reservation === undefined)\n      throw new Error(`trial ${trial} workspace was not reserved`);\n    return reservation;\n  }\n\n  semantic(trial: number): string | undefined {\n    return this.semantics[trial - 1];\n  }\n\n  retain(workspace: string): void {\n    this.retained.add(workspace);\n  }\n\n  completed(reservation: string): void {\n    this.retained.delete(reservation);\n  }\n\n  async cleanup(): Promise<void> {\n    for (const workspace of this.retained) await retainWorkspace(workspace);\n    await Promise.all(\n      [...this.candidates, ...this.semantics]\n        .filter((workspace) => !this.retained.has(workspace))\n        .map(async (workspace) => {\n          try {\n            await removeWorkspace(workspace);\n          } catch {\n            console.warn(`fixture cleanup failed; retained at ${workspace}`);\n          }\n        }),\n    );\n  }\n}\n\nexport interface RunResources {\n  readonly runId: string;\n  readonly runDir: string;\n  readonly runStateRoot: string;\n  readonly stateDir: string;\n  readonly evidencePath: string;\n  readonly owner: RunOwner;\n  readonly workspaces: RunWorkspaceReservations;\n}\n\nexport async function allocateRunResources(\n  options: Pick<EvaluationOptions, \"resultsRoot\" | \"runStateRoot\">,\n  evaluationDigest: string,\n): Promise<RunResources> {\n  const runId = randomUUID();\n  await mkdir(options.resultsRoot, { recursive: true, mode: 0o700 });\n  const runDir = resolve(await realpath(options.resultsRoot), runId);\n  await mkdir(runDir, { mode: 0o700 });\n  const runStateRoot = options.runStateRoot ?? options.resultsRoot;\n  await mkdir(runStateRoot, { recursive: true, mode: 0o700 });\n  const stateRoot = await realpath(runStateRoot);\n  const stateDir = join(stateRoot, runId);\n  if (stateDir !== runDir) await mkdir(stateDir, { mode: 0o700 });\n  const activePath = join(stateRoot, \"active\", `${runId}.json`);\n  const checkpointPath = join(stateDir, \"checkpoint.json\");\n  const evidencePath = join(runDir, \"run.json\");\n  let owner: RunOwner;\n  try {\n    owner = startRunOwner({\n      stateRoot,\n      evaluationDigest,\n      attemptId: runId,\n      activeRunPath: activePath,\n      checkpointPath,\n      evidenceDirectory: stateDir,\n      artifactPath: evidencePath,\n    });\n  } catch (error) {\n    await rm(runDir, { recursive: true, force: true });\n    if (stateDir !== runDir)\n      await rm(stateDir, { recursive: true, force: true });\n    throw error;\n  }\n  return {\n    runId,\n    runDir,\n    runStateRoot,\n    stateDir,\n    evidencePath,\n    owner,\n    workspaces: new RunWorkspaceReservations(),\n  };\n}\n\nexport async function retainPreparationArtifacts(\n  artifacts: PreparedFixture[\"inlineArtifacts\"],\n  runDir: string,\n): Promise<PreparationArtifactReference[]> {\n  const artifactRefs: PreparationArtifactReference[] = [];\n  for (const artifact of artifacts) {\n    const retainedPath = join(\n      runDir,\n      \"prepared\",\n      ...fixtureParts(artifact.relativePath),\n    );\n    await mkdir(dirname(retainedPath), { recursive: true, mode: 0o700 });\n    await writeFile(retainedPath, artifact.bytes, {\n      flag: \"wx\",\n      mode: artifact.executable ? 0o700 : 0o600,\n    });\n    artifactRefs.push({\n      id: artifact.id,\n      path: pathToFileURL(retainedPath).href,\n      sha256: artifact.sha256,\n      ...(artifact.gitExclude ? { gitExclude: true } : {}),\n      ...(artifact.executable ? { executable: true } : {}),\n    });\n  }\n  return artifactRefs;\n}\n\nfunction renderTrialPrompt(\n  prompt: string,\n  workspace: string,\n  invocation: PreparedInvocation,\n): string {\n  return prompt\n    .replaceAll(\"{{sevro.workspace}}\", workspace)\n    .replaceAll(\n      INVOCATION_PLACEHOLDER,\n      invocation.invocationToken ?? INVOCATION_PLACEHOLDER,\n    )\n    .replaceAll(\n      LEGACY_CODEX_PLACEHOLDER,\n      invocation.invocationToken ?? LEGACY_CODEX_PLACEHOLDER,\n    );\n}\n\ntype TrialPreparationOptions = Pick<\n  EvaluationOptions,\n  \"case\" | \"preparationSources\" | \"signal\"\n>;\n\nexport async function prepareTrialWorkspace(\n  options: TrialPreparationOptions,\n  fixture: PreparedFixture,\n  invocation: PreparedInvocation,\n  workspaces: RunWorkspaceReservations,\n  trial: number,\n  candidateRuntimeRoot?: string,\n) {\n  const reservation = workspaces.candidate(trial);\n  workspaces.retain(reservation);\n  const workspace = await createFixture(fixture.source, {\n    artifacts: fixture.inlineArtifacts,\n    sources: options.preparationSources,\n    setup: fixture.fixtureSetup,\n    projectRoot: fixture.projectRoot,\n    signal: options.signal,\n    reservedWorkspace: reservation,\n    candidateRuntimeRoot,\n  }).catch((error: unknown) => {\n    if (\n      !options.signal?.aborted ||\n      error instanceof EvaluationConfigurationError\n    )\n      throw error;\n    return null;\n  });\n  const details = await trialWorkspaceDetails(\n    options.case,\n    invocation,\n    reservation,\n    workspace,\n  );\n  return { ...details, candidateRuntimeRoot };\n}\n\nasync function trialWorkspaceDetails(\n  caseData: EvaluationOptions[\"case\"],\n  invocation: PreparedInvocation,\n  reservation: string,\n  preparedWorkspace: string | null,\n) {\n  const workspace = preparedWorkspace ?? reservation;\n  const fixtureBinDir =\n    preparedWorkspace === null\n      ? undefined\n      : ((await fixtureBinDirectory(workspace)) ?? undefined);\n  const trialPrompt = renderTrialPrompt(caseData.prompt, workspace, invocation);\n  const trialFollowUpPrompt =\n    caseData.followUpPrompt === undefined\n      ? undefined\n      : renderTrialPrompt(caseData.followUpPrompt, workspace, invocation);\n  return {\n    reservation,\n    workspace,\n    preparationCancelled: preparedWorkspace === null,\n    fixtureBinDir,\n    trialPrompt,\n    trialFollowUpPrompt,\n  };\n}\n"]}