@bjoernrochel/sevro 0.1.0-rc.15 → 0.1.0-rc.16

package/docs/extension-protocol-v1.md

@@ -106,8 +106,16 @@ trial in the Git fixture after source history and working-tree files are built,
 before extension artifacts are mounted or the host starts. The command is never
 interpreted through a shell by Sevro; an extension may explicitly select a
 shell executable. `{{sevro.project}}` and `{{sevro.workspace}}` in environment
-values resolve to the declared roots at execution time. The symbolic values and
-command enter fixture and comparison identity, while retained configuration
+values resolve to the declared roots at execution time. Setup's `HOME` is the
+candidate commands' private trial home, outside the Git fixture. Ordinary
+`$HOME` paths can initialize directories and seed prior state before the candidate
+starts, including without a runtime configuration file. This home is separate
+from host authentication and native configuration, other trials, and grader homes.
+Setup's `TMPDIR` remains disposable and is removed when setup finishes; the shared
+home survives through candidate execution and grading, then is removed even on
+setup failure, cancellation, or dry runs. Trusted injected hosts must use the
+supplied `candidateRuntimeRoot` and runtime policy to preserve this boundary.
+The symbolic values and command enter fixture and comparison identity, while retained configuration
 records only their digest. Setup has a two-minute limit, inherits no credential
 environment variables, and its output is discarded. Failure or interruption
 stops the run before host execution; it cannot become a passing assessment.

package/docs/installing.md

@@ -23,10 +23,10 @@ Local installation records the selected version in the project lockfile.
 
 ## Current release
 
-The current release is `0.1.0-rc.15`. Install this exact version:
+The current release is `0.1.0-rc.16`. Install this exact version:
 
 ```sh
-bun add --exact @bjoernrochel/sevro@0.1.0-rc.15
+bun add --exact @bjoernrochel/sevro@0.1.0-rc.16
 ```
 
 <!-- sevro-current-release:end -->

package/docs/runtime-v1.md

@@ -260,7 +260,10 @@ so zsh here-documents use their existing private temporary storage while shared
 host temporary directories remain denied. `TMPPREFIX` is runner-owned and cannot
 be inherited or set through runtime configuration.
 Sevro supplies private `HOME` and temporary storage and sets `UV_OFFLINE=1`
-and `PYTHONDONTWRITEBYTECODE=1`. Candidate, hook, checker, semantic, and advisory
+and `PYTHONDONTWRITEBYTECODE=1`. Fixture setup uses the candidate commands'
+private `HOME`, so `$HOME` paths initialize the same state the candidate will
+use. Its temporary directory stays disposable.
+Candidate, hook, checker, semantic, and advisory
 state are separate. With a runtime policy, native candidate, semantic, and
 advisory roots live in a unique runner-owned host directory outside the Git
 fixture. The root contains `home`, `tmp`, and declared seed slots; plugins can
@@ -270,7 +273,8 @@ configuration, and live transcript sources. Checker and hook caches also live
 outside candidate-controlled workspace state. Private roots survive native
 continuation turns and children, then are removed with their owning trial or
 host state after owned hooks terminate. Opted-in candidate artifact runtimes
-remain alive through grading as described below. Runs without a selected runtime policy
+remain alive through grading as described below. Runs with neither fixture setup
+nor a selected runtime policy
 retain their previous writable-state locations.
 
 Claude refuses managed workspace or runtime namespaces overlapping its
@@ -297,7 +301,8 @@ candidate's private home after execution.
 }
 ```
 
-The example permits only the candidate-created `~/.darrow/reviews` subtree.
+The example permits only the `~/.darrow/reviews` subtree, which may be initialized
+by trusted fixture setup or created by the candidate.
 Its original absolute path reaches isolated shell checks and native
 semantic/advisory commands as `SEVRO_CANDIDATE_ARTIFACT_REVIEWS`. Graders keep
 their own private `HOME`; use the supplied variable to read candidate evidence.
@@ -313,12 +318,31 @@ to 16,384 entries, including its root. Missing directories, symlinked ancestors,
 descendant symlinks, hardlinks and special files make grading an error; missing
 evidence cannot produce a pass.
 
+Initialize an empty directory or seed prior Review state in the extension's
+trusted `fixtureSetup` command before the candidate starts. For example:
+
+```sh
+mkdir -p "$HOME/.darrow/reviews"
+printf '%s\n' '{"verdict":"passed"}' > "$HOME/.darrow/reviews/prior.json"
+```
+
+Setup and candidate commands share this private trial home, including when no
+artifact grants or runtime file are selected. Graders keep their own homes and
+receive only declared read-only artifact grants. To test missing reports or
+absence of new artifacts, initialize the directory without writing a report;
+an empty safe directory reaches ordinary task checks. Missing required reports
+can then fail those checks, while negative cases can assert the directory is
+empty. Sevro does not automatically create declared artifact directories: if
+setup and candidate both leave one absent, or replace it with an unsafe path,
+grading still errors. Setup state is temporary and is removed with its trial,
+including setup failure, cancellation, and dry runs.
+
 The grant permits reads and denies writes, creation, deletion and renaming.
 Credentials, other home contents and peer role/trial state stay protected.
 Ordinary `filesystem.readOnlyRoots` cannot select this private-state exception.
 Artifact grants and native transcript access are independent opt-ins.
 
-The trial allocates the retained runtime before launching its candidate and
+The trial allocates the retained runtime before fixture setup and
 removes it after grading, advisory review and evidence persistence, including
 failure and cancellation paths. Review state remains outside the Git fixture.
 These artifacts are temporary; declaring access does not copy their bytes into

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bjoernrochel/sevro",
-  "version": "0.1.0-rc.15",
+  "version": "0.1.0-rc.16",
   "license": "SEE LICENSE IN LICENSE",
   "description": "A standalone evaluation runner for agentic coding capabilities",
   "type": "module",

package/src/candidate-artifacts.ts

@@ -65,10 +65,12 @@ function validArtifactPath(path: string): boolean {
 /** Allocate before host execution so exceptions cannot lose cleanup ownership. */
 export async function allocateCandidateRuntime(
   policy: RuntimePolicy | undefined,
+  retainHome = false,
 ): Promise<string | undefined> {
   const directories = policy?.artifactDirectories ?? [];
   const codexState = codexStateEnabled(policy);
-  if (!directories.length && !codexState) return undefined;
+  if (!candidateRuntimeRequired(directories, codexState, retainHome))
+    return undefined;
   requireArtifactDirectories(directories);
   const root = await allocateNativeState("candidate-runtime-");
   leases.set(root, directories);
@@ -76,6 +78,14 @@ export async function allocateCandidateRuntime(
   return root;
 }
 
+function candidateRuntimeRequired(
+  directories: RuntimeArtifactDirectory[],
+  codexState: boolean,
+  retainHome: boolean,
+): boolean {
+  return directories.length > 0 || codexState || retainHome;
+}
+
 export async function requireCandidateRuntime(
   root: string | undefined,
   role?: RuntimeRole,

package/src/evaluation/resources.ts

@@ -181,6 +181,7 @@ export async function prepareTrialWorkspace(
   invocation: PreparedInvocation,
   workspaces: RunWorkspaceReservations,
   trial: number,
+  candidateRuntimeRoot?: string,
 ) {
   const reservation = workspaces.candidate(trial);
   workspaces.retain(reservation);
@@ -195,6 +196,7 @@ export async function prepareTrialWorkspace(
     fixture.projectRoot,
     options.signal,
     reservation,
+    candidateRuntimeRoot,
   ).catch((error: unknown) => {
     if (
       !options.signal?.aborted ||
@@ -203,12 +205,13 @@ export async function prepareTrialWorkspace(
       throw error;
     return null;
   });
-  return trialWorkspaceDetails(
+  const details = await trialWorkspaceDetails(
     options.case,
     invocation,
     reservation,
     workspace,
   );
+  return { ...details, candidateRuntimeRoot };
 }
 
 async function trialWorkspaceDetails(

package/src/evaluation/trial.ts

@@ -231,6 +231,7 @@ class EvaluationTrial {
     private readonly fixture: Awaited<ReturnType<typeof prepareTrialWorkspace>>,
   ) {
     this.state = initialTrialState(options.dry);
+    this.state.candidateRuntimeRoot = fixture.candidateRuntimeRoot;
     this.evidence = new TrialEvidenceWriter({
       runDir: context.resources.runDir,
       trial,
@@ -296,7 +297,7 @@ class EvaluationTrial {
       workspace: this.fixture.workspace,
       condition: this.options.condition,
       fixtureBinDir: this.fixture.fixtureBinDir,
-      runtimePolicy: this.options.runtimePolicy,
+      runtimePolicy: this.candidateRuntimePolicy(),
       instrumentation: snapshotInstrumentation(
         this.context.prepared.requestedInstrumentation,
       ),
@@ -313,6 +314,16 @@ class EvaluationTrial {
     };
   }
 
+  private candidateRuntimePolicy() {
+    if (this.options.runtimePolicy) return this.options.runtimePolicy;
+    if (!this.context.prepared.fixture.fixtureSetup) return undefined;
+    return {
+      format: "sevro.runtime.v1" as const,
+      environment: {},
+      readOnlyRoots: [],
+    };
+  }
+
   private candidateReservedIds(): Set<string> {
     return new Set([
       "sevro.observation.final-message",
@@ -387,9 +398,6 @@ class EvaluationTrial {
     const started = performance.now();
     try {
       ensureNotCancelled(this.options.signal);
-      this.state.candidateRuntimeRoot = await allocateCandidateRuntime(
-        this.options.runtimePolicy,
-      );
       this.state.hostResult = await this.options.host.run(
         this.candidateRequest(),
       );
@@ -781,7 +789,6 @@ class EvaluationTrial {
     evaluationProgress(this.options, stage, status, this.trial);
   }
   private async cleanup(): Promise<void> {
-    await releaseCandidateRuntime(this.state.candidateRuntimeRoot);
     if (this.state.candidateTranscriptRoot)
       await rm(this.state.candidateTranscriptRoot, {
         recursive: true,
@@ -819,21 +826,30 @@ export async function executeEvaluationTrials(
       signal: options.signal,
     },
     async (trial, stopAdmission) => {
-      const fixture = await prepareTrialWorkspace(
-        options,
-        context.prepared.fixture,
-        context.prepared.invocation,
-        context.resources.workspaces,
-        trial,
+      const candidateRuntimeRoot = await allocateCandidateRuntime(
+        options.runtimePolicy,
+        Boolean(context.prepared.fixture.fixtureSetup),
       );
-      evaluationProgress(options, "preparation", "completed", trial);
-      return new EvaluationTrial(
-        options,
-        context,
-        trial,
-        stopAdmission,
-        fixture,
-      ).run();
+      try {
+        const fixture = await prepareTrialWorkspace(
+          options,
+          context.prepared.fixture,
+          context.prepared.invocation,
+          context.resources.workspaces,
+          trial,
+          candidateRuntimeRoot,
+        );
+        evaluationProgress(options, "preparation", "completed", trial);
+        return await new EvaluationTrial(
+          options,
+          context,
+          trial,
+          stopAdmission,
+          fixture,
+        ).run();
+      } finally {
+        await releaseCandidateRuntime(candidateRuntimeRoot);
+      }
     },
   );
 }

package/src/fixtures/create.ts

@@ -36,6 +36,7 @@ export async function createFixture(
   projectRoot: string,
   signal?: AbortSignal,
   reservedWorkspace?: string,
+  candidateRuntimeRoot?: string,
 ): Promise<string> {
   const paths = inlineFixturePaths(fixture, generated, repository);
   const workspace = reservedWorkspace ?? (await allocateWorkspace());
@@ -53,7 +54,13 @@ export async function createFixture(
       workspace,
     );
     await writeInlineFixtureFiles(paths, workspace);
-    await setupFixture(setup, workspace, projectRoot, signal);
+    await setupFixture(
+      setup,
+      workspace,
+      projectRoot,
+      signal,
+      candidateRuntimeRoot,
+    );
     await mountPreparationArtifacts(artifacts, workspace);
     await excludePreparationArtifacts(artifacts, workspace);
     await fixtureBinDirectory(workspace);
@@ -131,6 +138,7 @@ async function setupFixture(
   workspace: string,
   projectRoot: string,
   signal?: AbortSignal,
+  candidateRuntimeRoot?: string,
 ): Promise<void> {
   if (!setup) return;
   await runFixtureSetup(setup, {
@@ -138,6 +146,7 @@ async function setupFixture(
     projectRoot,
     fixtureBinDir: (await fixtureBinDirectory(workspace)) ?? undefined,
     signal,
+    candidateRuntimeRoot,
   });
 }
 

package/src/fixtures/setup.ts

@@ -1,7 +1,8 @@
-import { mkdtemp, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { delimiter, isAbsolute, join } from "node:path";
 import { isRecord } from "../value-guards";
+import { requireCandidateRuntime } from "../candidate-artifacts";
 import {
   ProcessDeadline,
   terminateOwnedProcessGroup,
@@ -112,6 +113,7 @@ interface FixtureSetupOptions {
   projectRoot: string;
   fixtureBinDir?: string;
   signal?: AbortSignal;
+  candidateRuntimeRoot?: string;
 }
 
 function setupPath(directory: string | undefined): string {
@@ -134,7 +136,9 @@ function executionEnvironment(
   );
   return {
     PATH: setupPath(options.fixtureBinDir),
-    HOME: home,
+    HOME: options.candidateRuntimeRoot
+      ? join(options.candidateRuntimeRoot, "home")
+      : home,
     TMPDIR: home,
     LANG: process.env.LANG ?? "C",
     GIT_CONFIG_GLOBAL: process.platform === "win32" ? "NUL" : "/dev/null",
@@ -201,6 +205,12 @@ export async function runFixtureSetup(
   options: FixtureSetupOptions,
 ): Promise<void> {
   if (options.signal?.aborted) throw new Error("fixture setup cancelled");
+  await requireCandidateRuntime(options.candidateRuntimeRoot);
+  if (options.candidateRuntimeRoot)
+    await mkdir(join(options.candidateRuntimeRoot, "home"), {
+      recursive: true,
+      mode: 0o700,
+    });
   const home = await mkdtemp(join(tmpdir(), "sevro-setup-"));
   const execution = new FixtureSetupExecution(setup, options, home);
   try {