@bjoernrochel/sevro 0.1.0-rc.7 → 1.0.0

package/src/{evaluation-semantic.ts → evaluation/semantic.ts}

@@ -1,50 +1,53 @@
-import { rm } from "node:fs/promises";
-import { join } from "node:path";
-import { pathToFileURL } from "node:url";
-import { createFixture, hostArtifacts, usage } from "./evaluation-fixture";
+import { clearFixtureContents } from "../fixtures/cleanup";
+import type { CandidateArtifactRoots } from "../candidate-artifacts";
+import { createFixture } from "../fixtures/create";
+import { hostArtifacts } from "../evidence/host-results";
+import { usage } from "../evidence/usage";
+import { gradingRuntimePolicy, scopedHostArtifacts } from "./grading-host";
 import {
-  gradingRuntimePolicy,
-  scopedHostArtifacts,
-} from "./evaluation-grading-host";
-import type { HostAdapter, HostResult, ResolvedCase } from "./evaluation-types";
-import { readSemanticArtifact } from "./graders/artifact";
+  HostExecutionFailure,
+  type HostAdapter,
+  type HostResult,
+} from "./types";
+import { readSemanticArtifact } from "../graders/artifact";
 import {
   parseSemanticVerdicts,
   semanticCheckGroups,
   semanticPrompt,
   type PreparedSemanticCheck,
   type SemanticVerdict,
-} from "./graders/semantic";
-import { sha256 } from "./identity";
-import type { CheckOutcome } from "./results";
-import type { RuntimePolicy } from "./runtime-config";
+} from "../graders/semantic";
+import { sha256 } from "../identity";
+import type { CheckOutcome } from "../results";
+import type {
+  TrialEvidenceWriter,
+  TrialObservation,
+} from "../evidence/trial-writer";
+import type { GradingFailure, GradingResult } from "./grading";
+import type { RuntimePolicy } from "../runtime/config";
+import { HostDiagnosticPersistenceError } from "../host-diagnostics";
 
-interface SemanticObservation {
-  id: string;
-  source: string;
-  completeness: "complete" | "partial";
-  data: Record<string, unknown>;
-}
 interface SemanticGradingInput {
-  checks: PreparedSemanticCheck[];
-  declaredChecks: ResolvedCase["checks"];
-  host: HostAdapter | undefined;
-  message: string | null;
-  candidateWorkspace: string;
-  candidateTranscriptRoot?: string;
-  projectRoot: string;
-  reservedWorkspace?: string;
-  runDir: string;
-  trial: number;
-  runtimePolicy?: RuntimePolicy;
-  signal?: AbortSignal;
+  readonly checks: readonly PreparedSemanticCheck[];
+  readonly declaredChecks: readonly { id: string }[];
+  readonly host: HostAdapter | undefined;
+  readonly message: string | null;
+  readonly candidateWorkspace: string;
+  readonly candidateTranscriptRoot?: string;
+  readonly candidateArtifactRoots?: CandidateArtifactRoots;
+  readonly projectRoot: string;
+  readonly reservedWorkspace?: string;
+  readonly runtimePolicy?: RuntimePolicy;
+  readonly signal?: AbortSignal;
 }
 interface SemanticGradingEffects {
-  artifactRefs: { id: string; path: string; sha256: string }[];
-  canGrade(): boolean;
-  failGrading(message: string, cancelable?: boolean): void;
-  retainArtifacts(artifacts: ReturnType<typeof hostArtifacts>): Promise<void>;
-  writeTrialFile(path: string, bytes: string | Uint8Array): Promise<void>;
+  readonly evidence: Pick<
+    TrialEvidenceWriter,
+    "artifactRefs" | "retainArtifacts" | "retainSemanticVerdicts"
+  >;
+  /** Close admission at detection, before retaining or cleaning auxiliary state. */
+  onFailure(failure: GradingFailure): void;
+  retainWorkspace(workspace: string): Promise<void>;
 }
 type SemanticGroup = ReturnType<typeof semanticCheckGroups>[number];
 
@@ -94,16 +97,25 @@ function semanticHostPrompt(
   );
 }
 class SemanticGrading {
-  private observations: SemanticObservation[] = [];
+  private observations: TrialObservation[] = [];
   private checks: CheckOutcome[] = [];
+  private failure: GradingFailure | null = null;
   constructor(
     private readonly input: SemanticGradingInput,
     private readonly effects: SemanticGradingEffects,
   ) {}
-  async run() {
-    for (const group of semanticCheckGroups(this.input.checks))
+  async run(): Promise<GradingResult> {
+    for (const group of semanticCheckGroups([...this.input.checks]))
       await this.gradeSemanticGroup(group);
-    return { checks: this.checks, observations: this.observations };
+    return {
+      checks: this.checks,
+      observations: this.observations,
+      failure: this.failure,
+    };
+  }
+  private failGrading(message: string, cancelable = true): void {
+    this.failure = { message, cancelable };
+    this.effects.onFailure(this.failure);
   }
   private async semanticInput(
     pattern: string | undefined,
@@ -115,16 +127,12 @@ class SemanticGrading {
 
   private async semanticWorkspace(): Promise<string> {
     return createFixture(
-      { files: {} },
-      [],
-      undefined,
-      null,
-      null,
-      null,
-      null,
-      this.input.projectRoot,
-      this.input.signal,
-      this.input.reservedWorkspace,
+      { kind: "inline", files: {} },
+      {
+        projectRoot: this.input.projectRoot,
+        signal: this.input.signal,
+        reservedWorkspace: this.input.reservedWorkspace,
+      },
     );
   }
 
@@ -141,53 +149,43 @@ class SemanticGrading {
       source = semanticSource(artifact);
       const host = this.input.host;
       if (!host) throw new Error("semantic host is unavailable");
-      const response = await host.run({
-        candidateTranscriptRoot: this.input.candidateTranscriptRoot,
-        prompt: semanticHostPrompt(message, group, artifact),
-        runtimePolicy: gradingRuntimePolicy(
-          this.input.runtimePolicy,
-          pattern === undefined,
-        ),
-        runtimeRole: "semantic",
-        workspace,
-        condition: "passive",
-        signal: this.input.signal,
-      });
+      const response = await host
+        .run({
+          retainHostDiagnostics: (bytes) =>
+            this.effects.evidence.retainArtifacts([
+              {
+                id: `sevro.semantic${suffix}.sevro.host.diagnostics`,
+                bytes: Buffer.from(bytes),
+              },
+            ]),
+          candidateTranscriptRoot: this.input.candidateTranscriptRoot,
+          candidateArtifactRoots: this.input.candidateArtifactRoots,
+          prompt: semanticHostPrompt(message, group, artifact),
+          runtimePolicy: gradingRuntimePolicy(
+            this.input.runtimePolicy,
+            pattern === undefined,
+          ),
+          runtimeRole: "semantic",
+          workspace,
+          condition: "passive",
+          signal: this.input.signal,
+        })
+        .catch((cause: unknown) => {
+          if (cause instanceof HostExecutionFailure) return cause.result;
+          throw cause;
+        });
       const artifacts = scopedHostArtifacts(
         response,
         `sevro.semantic${suffix}.`,
-        this.effects.artifactRefs,
+        this.effects.evidence.artifactRefs,
         this.input.declaredChecks,
         `sevro.semantic.verdicts${suffix}`,
       );
       return { result: response, artifacts, source, hostId: host.id };
-    } catch {
-      this.effects.failGrading("semantic grading did not complete");
+    } catch (cause) {
+      if (cause instanceof HostDiagnosticPersistenceError) throw cause;
+      this.failGrading("semantic grading did not complete");
       return { result: null, artifacts: [], source, hostId: null };
-    } finally {
-      await rm(workspace, { recursive: true, force: true });
-    }
-  }
-
-  private async retainSemanticVerdicts(
-    result: HostResult,
-    suffix: string,
-  ): Promise<void> {
-    if (
-      result.finalMessage !== null &&
-      Buffer.byteLength(result.finalMessage, "utf8") <= 1024 * 1024
-    ) {
-      const bytes = Buffer.from(result.finalMessage, "utf8");
-      const path = join(
-        this.input.runDir,
-        `trial-${this.input.trial}-semantic-verdicts${suffix}.json`,
-      );
-      await this.effects.writeTrialFile(path, bytes);
-      this.effects.artifactRefs.push({
-        id: `sevro.semantic.verdicts${suffix}`,
-        path: pathToFileURL(path).href,
-        sha256: sha256(bytes),
-      });
     }
   }
 
@@ -223,14 +221,19 @@ class SemanticGrading {
     pattern: string | undefined,
     source: Record<string, unknown>,
     hostId: string,
-  ): void {
+  ): boolean {
     try {
       const message = this.completeSemanticGraderMessage(result);
       const verdicts = parseSemanticVerdicts(message, group);
       for (const verdict of verdicts)
         this.recordSemanticVerdict(verdict, pattern, source, hostId);
+      return true;
     } catch {
-      this.effects.failGrading("semantic grading did not complete", false);
+      this.failGrading(
+        result.failureMessage ?? "semantic grading did not complete",
+        result.processFailure?.reason === "cancelled",
+      );
+      return false;
     }
   }
 
@@ -243,7 +246,7 @@ class SemanticGrading {
   private async gradeSemanticGroup(group: SemanticGroup): Promise<void> {
     const pattern = group[0].artifactPath;
     const suffix = semanticGroupSuffix(pattern);
-    if (!this.effects.canGrade()) return;
+    if (this.failure) return;
     const message = this.input.message;
     if (message === null) {
       this.checks.push(
@@ -257,29 +260,44 @@ class SemanticGrading {
       return;
     }
     const workspace = await this.semanticWorkspace();
-    const outcome = await this.runSemanticHost(
-      group,
-      pattern,
-      suffix,
-      workspace,
-      message,
-    );
-    if (outcome.result === null) return;
-    await this.effects.retainArtifacts(outcome.artifacts);
-    this.observations.push({
-      id: `sevro.observation.semantic.usage${suffix}`,
-      source: outcome.hostId,
-      completeness: outcome.result.usageComplete ? "complete" : "partial",
-      data: usage(outcome.result),
-    });
-    await this.retainSemanticVerdicts(outcome.result, suffix);
-    this.parseSemanticResults(
-      outcome.result,
-      group,
-      pattern,
-      outcome.source,
-      outcome.hostId,
-    );
+    let successful = false;
+    try {
+      const outcome = await this.runSemanticHost(
+        group,
+        pattern,
+        suffix,
+        workspace,
+        message,
+      );
+      if (outcome.result === null) return;
+      await this.effects.evidence.retainArtifacts(outcome.artifacts);
+      this.observations.push({
+        id: `sevro.observation.semantic.usage${suffix}`,
+        source: outcome.hostId,
+        completeness: outcome.result.usageComplete ? "complete" : "partial",
+        data: usage(outcome.result),
+      });
+      await this.effects.evidence.retainSemanticVerdicts(
+        outcome.result.finalMessage,
+        suffix,
+      );
+      successful = this.parseSemanticResults(
+        outcome.result,
+        group,
+        pattern,
+        outcome.source,
+        outcome.hostId,
+      );
+    } finally {
+      await this.finishWorkspace(workspace, successful);
+    }
+  }
+  private async finishWorkspace(
+    workspace: string,
+    successful: boolean,
+  ): Promise<void> {
+    if (successful) await clearFixtureContents(workspace);
+    else await this.effects.retainWorkspace(workspace);
   }
 }
 export async function gradeSemanticChecks(