@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0

package/src/{evaluation-types.ts → evaluation/types.ts}

@@ -1,16 +1,26 @@
 import type {
   InstrumentationCapability,
   InstrumentationRequest,
-} from "./instrumentation";
-import type { GeneratedFixture } from "./generated-fixture";
-import type { PreparationSources } from "./preparation";
-import type { RepositoryFixture } from "./repository-fixture";
-import type { EvaluationResult, ExtensionCase } from "./extension-session";
-import type { openExtensionSession } from "./extension-session";
-import type { Assessment, CheckOutcome } from "./results";
-import type { RuntimePolicy } from "./runtime-config";
-import type { RuntimeRole } from "./runtime-state";
-import type { CandidateArtifactRoots } from "./candidate-artifacts";
+} from "../instrumentation";
+import type { GeneratedFixture } from "../fixtures/generated";
+import type { PreparationSources } from "../fixtures/preparation";
+import type { RepositoryFixture } from "../fixtures/repository";
+import type { ExtensionCase } from "../extension-session";
+import type { openExtensionSession } from "../extension-session";
+import type {
+  Assessment,
+  CheckDeclaration,
+  DomainOutcome,
+  ObservationEvidence,
+} from "../contract-types";
+import type {
+  CliResultData,
+  CliResultDataCase,
+  CliResultDataTrial,
+} from "../schema-types";
+import type { RuntimePolicy } from "../runtime/config";
+import type { RuntimeRole } from "../runtime/state";
+import type { CandidateArtifactRoots } from "../candidate-artifacts";
 
 export class EvaluationConfigurationError extends Error {
   constructor(message: string, options?: ErrorOptions) {
@@ -23,17 +33,15 @@ export interface HostResult {
   finalMessage: string | null;
   complete: boolean;
   executionFailed?: boolean;
+  /** Adapter-owned, bounded failure summary suitable for CLI diagnostics. */
+  failureMessage?: string;
   processFailure?: {
     phase: "isolation-preflight" | "executable-preflight" | "execution";
     reason: "exit" | "timeout" | "cancelled";
     timeoutMs: number;
   };
   artifacts?: { id: string; bytes: Uint8Array }[];
-  observations?: {
-    id: string;
-    completeness: "complete" | "partial" | "unavailable";
-    data: Record<string, unknown>;
-  }[];
+  observations?: Omit<ObservationEvidence, "source">[];
   actualCondition?: "passive" | "enforced" | "unknown";
   appliedInstrumentation?: InstrumentationRequest[];
   inputTokens?: number | null;
@@ -42,6 +50,28 @@ export interface HostResult {
   usageComplete?: boolean;
 }
 
+/** A failed host call can still provide bounded evidence for retention. */
+export class HostExecutionFailure extends Error {
+  readonly result: HostResult;
+  constructor(
+    message: string | undefined,
+    result: HostResult,
+    options?: ErrorOptions,
+  ) {
+    super(message, options);
+    this.result = { ...result, executionFailed: true, failureMessage: message };
+  }
+}
+
+export class TrialPersistenceError extends Error {
+  constructor(workspace: string, options?: ErrorOptions) {
+    super(
+      `trial persistence failed; fixture retained at ${workspace}`,
+      options,
+    );
+  }
+}
+
 export interface HostAdapter {
   id: string;
   model: string;
@@ -50,6 +80,8 @@ export interface HostAdapter {
   hostCapabilities?: string[];
   configuration?: Record<string, unknown>;
   run(request: {
+    /** Persist private diagnostic bytes before the native host removes its state. */
+    retainHostDiagnostics?: (bytes: Uint8Array) => Promise<void>;
     runtimePolicy?: RuntimePolicy;
     runtimeRole?: RuntimeRole;
     candidateTranscriptRoot?: string;
@@ -96,15 +128,23 @@ export interface ResolvedCase {
     | { files: Record<string, string>; sourceRef?: never }
     | Omit<RepositoryFixture, "kind">
     | (GeneratedFixture & { sourceRef?: never });
-  checks: {
-    id: string;
-    grader: string;
-    configuration: Record<string, unknown>;
-  }[];
+  checks: CheckDeclaration[];
   requiredEvidence: string[];
 }
 
+export interface EvaluationProgress {
+  stage: "preparation" | "candidate" | "grading" | "evidence";
+  status: "started" | "completed" | "skipped";
+  trial?: number;
+}
+
 export interface EvaluationOptions {
+  /** Optional best-effort observation; exceptions never change the evaluation. */
+  onProgress?: (event: EvaluationProgress) => void;
+  /** Redacted operator preview, retained for reproduction outside semantic identity. */
+  resolvedRunSettings?: Record<string, unknown>;
+  /** Additional non-secret assessment settings included in configuration identity. */
+  assessmentSettings?: Record<string, unknown>;
   runtimePolicy?: RuntimePolicy;
   projectRoot: string;
   resultsRoot: string;
@@ -135,23 +175,33 @@ export interface EvaluationOptions {
   };
 }
 
-export interface TrialSummary extends Assessment {
-  trial: number;
-  checks: CheckOutcome[];
-  domainOutcomes: NonNullable<EvaluationResult["domainOutcomes"]>;
+/** Persisted evaluations always retain a trial path and a known outcome list. */
+export interface TrialSummary
+  extends
+    Assessment,
+    Omit<
+      CliResultDataTrial,
+      keyof Assessment | "domainOutcomes" | "artifactPath"
+    > {
+  domainOutcomes: DomainOutcome[];
   artifactPath: string;
 }
 
-export interface CaseSummary extends Assessment {
-  caseId: string;
+export interface CaseSummary
+  extends Assessment, Omit<CliResultDataCase, keyof Assessment | "trials"> {
   trials: TrialSummary[];
 }
 
-export interface CliResult extends Assessment {
-  format: "sevro.cli-result.v1";
+/** Unlike pre-run wire failures, a returned evaluation has run/evidence paths. */
+export interface CliResult
+  extends
+    Assessment,
+    Omit<
+      CliResultData,
+      keyof Assessment | "runId" | "exitCode" | "evidencePath" | "cases"
+    > {
   runId: string;
   exitCode: number;
   evidencePath: string;
   cases: CaseSummary[];
-  diagnostic?: { code: string; message: string };
 }

package/src/evidence/host-results.ts

@@ -0,0 +1,85 @@
+import { isRecord, validQualifiedId as validEvidenceId } from "../value-guards";
+import { canonicalJson } from "../identity";
+import type { HostResult } from "../evaluation/types";
+import type {
+  ObservationCompleteness,
+  ObservationEvidence,
+} from "../contract-types";
+
+export function hostObservations(
+  result: HostResult,
+  hostId: string,
+  existingIds: Set<string>,
+): ObservationEvidence[] {
+  const items = result.observations ?? [];
+  if (items.length > 128) throw new Error("too many host observations");
+  const ids = new Set([...existingIds, "sevro.observation.final-message"]);
+  let totalBytes = 0;
+  return items.map((value: unknown) => {
+    const item = validatedHostObservation(value, ids);
+    ids.add(item.id);
+    const data = canonicalJson(item.data);
+    totalBytes += Buffer.byteLength(data, "utf8");
+    if (totalBytes > 8 * 1024 * 1024)
+      throw new Error("host observations exceed 8 MiB");
+    return {
+      id: item.id,
+      source: hostId,
+      completeness: item.completeness,
+      data: JSON.parse(data) as Record<string, unknown>,
+    };
+  });
+}
+
+export function hostArtifacts(result: HostResult, existingIds: Set<string>) {
+  const items = result.artifacts ?? [];
+  if (!Array.isArray(items) || items.length > 32)
+    throw new Error("too many host artifacts");
+  const ids = new Set(existingIds);
+  let totalBytes = 0;
+  return items.map((value: unknown) => {
+    const item = validatedHostArtifact(value, ids);
+    ids.add(item.id);
+    totalBytes += item.bytes.byteLength;
+    if (
+      item.bytes.byteLength > 8 * 1024 * 1024 ||
+      totalBytes > 32 * 1024 * 1024
+    )
+      throw new Error("host artifacts exceed the size limit");
+    return { id: item.id, bytes: Buffer.from(item.bytes) };
+  });
+}
+
+function validObservationId(value: unknown, ids: Set<string>): value is string {
+  return (
+    validEvidenceId(value) &&
+    !value.startsWith("sevro.observation.") &&
+    !ids.has(value)
+  );
+}
+
+function validCompleteness(value: unknown): value is ObservationCompleteness {
+  return value === "complete" || value === "partial" || value === "unavailable";
+}
+
+function validatedHostObservation(value: unknown, ids: Set<string>) {
+  if (!isRecord(value)) throw new Error("invalid host observation");
+  if (
+    !validObservationId(value.id, ids) ||
+    !validCompleteness(value.completeness) ||
+    !isRecord(value.data)
+  )
+    throw new Error("invalid host observation");
+  return { id: value.id, completeness: value.completeness, data: value.data };
+}
+
+function validArtifactId(value: unknown, ids: Set<string>): value is string {
+  return validEvidenceId(value) && !ids.has(value);
+}
+
+function validatedHostArtifact(value: unknown, ids: Set<string>) {
+  if (!isRecord(value)) throw new Error("invalid host artifact");
+  if (!validArtifactId(value.id, ids) || !(value.bytes instanceof Uint8Array))
+    throw new Error("invalid host artifact");
+  return { id: value.id, bytes: value.bytes };
+}

package/src/evidence/retained-artifacts.ts

@@ -0,0 +1,18 @@
+import { readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { sha256 } from "../identity";
+
+export async function verifyRetainedArtifacts(
+  artifacts: { path: string; sha256: string }[],
+): Promise<void> {
+  for (const artifact of artifacts) {
+    let actual: string;
+    try {
+      actual = sha256(await readFile(fileURLToPath(artifact.path)));
+    } catch {
+      throw new Error("retained preparation artifact is unreadable");
+    }
+    if (actual !== artifact.sha256)
+      throw new Error("retained preparation artifact changed");
+  }
+}

package/src/evidence/trial-writer.ts

@@ -0,0 +1,203 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { TrialPersistenceError, type TrialSummary } from "../evaluation/types";
+import { sha256 } from "../identity";
+import type { Assessment, CheckOutcome } from "../results";
+import type {
+  ArtifactReference,
+  Diagnostic,
+  ObservationEvidence,
+  RawResultReference,
+} from "../contract-types";
+import type { RunEvidenceDataTrialEvidence } from "../schema-types";
+import { atomicWriteJson } from "../storage";
+import type { hostArtifacts } from "./host-results";
+import { verifyRetainedArtifacts } from "./retained-artifacts";
+
+type TrialEvidence = RunEvidenceDataTrialEvidence;
+export type TrialObservation = ObservationEvidence;
+
+interface TrialWriterInput {
+  readonly runDir: string;
+  readonly trial: number;
+  readonly workspace: string;
+  readonly preparationArtifacts: readonly ArtifactReference[];
+}
+
+/** Owns private trial files and their references, independently of run collection. */
+export class TrialEvidenceWriter {
+  private readonly references: ArtifactReference[];
+  private rawPath: string | null = null;
+  private rawDigest: string | null = null;
+
+  constructor(private readonly input: TrialWriterInput) {
+    this.references = [...input.preparationArtifacts];
+  }
+
+  get artifactRefs(): ArtifactReference[] {
+    return [...this.references];
+  }
+
+  get hasCandidateMessage(): boolean {
+    return this.rawDigest !== null;
+  }
+
+  private async writePrivateFile(
+    path: string,
+    bytes: string | Uint8Array,
+  ): Promise<void> {
+    try {
+      await writeFile(path, bytes, { flag: "wx", mode: 0o600 });
+    } catch (cause) {
+      throw new TrialPersistenceError(this.input.workspace, { cause });
+    }
+  }
+
+  private async retainReference(
+    id: string,
+    filename: string,
+    bytes: string | Uint8Array,
+  ): Promise<ArtifactReference> {
+    const path = join(
+      this.input.runDir,
+      `trial-${this.input.trial}-${filename}`,
+    );
+    await this.writePrivateFile(path, bytes);
+    const reference = {
+      id,
+      path: pathToFileURL(path).href,
+      sha256: sha256(bytes),
+    };
+    this.references.push(reference);
+    return reference;
+  }
+
+  async retainArtifacts(
+    artifacts: readonly ReturnType<typeof hostArtifacts>[number][],
+    prefix = "",
+  ): Promise<void> {
+    for (const artifact of artifacts)
+      await this.retainReference(
+        artifact.id,
+        `${prefix}${artifact.id}.bin`,
+        artifact.bytes,
+      );
+  }
+
+  verifyArtifacts(): Promise<void> {
+    return verifyRetainedArtifacts(this.references);
+  }
+
+  observeCandidate(
+    source: string,
+    result: { finalMessage: string | null; complete: boolean } | null,
+  ): TrialObservation {
+    if (result === null || result.finalMessage == null) {
+      this.rawDigest = null;
+      return {
+        id: "sevro.observation.final-message",
+        source,
+        completeness: "unavailable",
+        data: {},
+      };
+    }
+    this.rawDigest = sha256(result.finalMessage);
+    return {
+      id: "sevro.observation.final-message",
+      source,
+      completeness: result.complete ? "complete" : "partial",
+      data: {
+        sha256: this.rawDigest,
+        byteLength: Buffer.byteLength(result.finalMessage, "utf8"),
+      },
+    };
+  }
+
+  async retainSemanticVerdicts(
+    message: string | null,
+    suffix: string,
+  ): Promise<void> {
+    if (message === null || Buffer.byteLength(message, "utf8") > 1024 * 1024)
+      return;
+    await this.retainReference(
+      `sevro.semantic.verdicts${suffix}`,
+      `semantic-verdicts${suffix}.json`,
+      Buffer.from(message, "utf8"),
+    );
+  }
+
+  async retainAdvisoryResponse(
+    source: string,
+    message: string | null,
+  ): Promise<RawResultReference> {
+    if (message === null || Buffer.byteLength(message, "utf8") > 64 * 1024)
+      return { source, path: null, sha256: null };
+    const reference = await this.retainReference(
+      "sevro.advisory.response",
+      "advisory.json",
+      message,
+    );
+    return { source, path: reference.path, sha256: reference.sha256 };
+  }
+
+  async retainRawCandidate(message: string | null | undefined): Promise<void> {
+    await this.verifyArtifacts();
+    if (message == null) {
+      this.rawPath = null;
+      return;
+    }
+    this.rawPath = join(this.input.runDir, `trial-${this.input.trial}-raw.txt`);
+    await this.writePrivateFile(this.rawPath, message);
+  }
+
+  candidateRawResult(source: string): RawResultReference {
+    return {
+      source,
+      path: this.rawPath ? pathToFileURL(this.rawPath).href : null,
+      sha256: this.rawDigest,
+    };
+  }
+
+  diagnosticLocation(diagnostic: Diagnostic | undefined) {
+    if (!diagnostic) return diagnostic;
+    const artifact = this.references.find((ref) =>
+      ref.id.endsWith("sevro.host.diagnostics"),
+    );
+    if (!artifact) return diagnostic;
+    const message = `${diagnostic.message}; diagnostics: ${artifact.path}`;
+    return message.length <= 4096 ? { ...diagnostic, message } : diagnostic;
+  }
+
+  async persistTrial(
+    caseId: string,
+    assessment: Assessment,
+    checks: CheckOutcome[],
+    domainOutcomes: TrialSummary["domainOutcomes"],
+    evidence: TrialEvidence,
+  ): Promise<TrialSummary> {
+    const artifactPath = join(
+      this.input.runDir,
+      `trial-${this.input.trial}.json`,
+    );
+    const summary: TrialSummary = {
+      trial: this.input.trial,
+      ...assessment,
+      checks,
+      domainOutcomes,
+      artifactPath,
+    };
+    try {
+      await atomicWriteJson(artifactPath, {
+        format: "sevro.trial-evidence.v1",
+        caseId,
+        trial: this.input.trial,
+        result: summary,
+        evidence,
+      });
+    } catch (cause) {
+      throw new TrialPersistenceError(this.input.workspace, { cause });
+    }
+    return summary;
+  }
+}

package/src/evidence/usage.ts

@@ -0,0 +1,17 @@
+import type { HostResult } from "../evaluation/types";
+import type { UsageEvidence } from "../contract-types";
+
+export function usage(result: HostResult | null): UsageEvidence {
+  return result === null
+    ? { inputTokens: null, outputTokens: null, costUsd: null, complete: false }
+    : availableUsage(result);
+}
+
+function availableUsage(result: HostResult): UsageEvidence {
+  return {
+    inputTokens: result.inputTokens ?? null,
+    outputTokens: result.outputTokens ?? null,
+    costUsd: result.costUsd ?? null,
+    complete: result.usageComplete ?? false,
+  };
+}

package/src/executable.ts

@@ -0,0 +1,19 @@
+import { accessSync, constants, statSync } from "node:fs";
+import { delimiter, resolve } from "node:path";
+
+export function which(name: string, path?: string): string | null {
+  for (const directory of executablePath(path).split(delimiter)) {
+    const candidate = resolve(directory, name);
+    try {
+      accessSync(candidate, constants.X_OK);
+      if (statSync(candidate).isFile()) return candidate;
+    } catch {
+      /* Continue searching the declared executable path. */
+    }
+  }
+  return null;
+}
+
+function executablePath(path: string | undefined): string {
+  return path ?? process.env.PATH ?? "";
+}