@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/evaluation/types.ts

@@ -0,0 +1,207 @@
+import type {
+  InstrumentationCapability,
+  InstrumentationRequest,
+} from "../instrumentation";
+import type { GeneratedFixture } from "../fixtures/generated";
+import type { PreparationSources } from "../fixtures/preparation";
+import type { RepositoryFixture } from "../fixtures/repository";
+import type { ExtensionCase } from "../extension-session";
+import type { openExtensionSession } from "../extension-session";
+import type {
+  Assessment,
+  CheckDeclaration,
+  DomainOutcome,
+  ObservationEvidence,
+} from "../contract-types";
+import type {
+  CliResultData,
+  CliResultDataCase,
+  CliResultDataTrial,
+} from "../schema-types";
+import type { RuntimePolicy } from "../runtime/config";
+import type { RuntimeRole } from "../runtime/state";
+import type { CandidateArtifactRoots } from "../candidate-artifacts";
+
+export class EvaluationConfigurationError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+    this.name = "EvaluationConfigurationError";
+  }
+}
+
+export interface HostResult {
+  finalMessage: string | null;
+  complete: boolean;
+  executionFailed?: boolean;
+  /** Adapter-owned, bounded failure summary suitable for CLI diagnostics. */
+  failureMessage?: string;
+  processFailure?: {
+    phase: "isolation-preflight" | "executable-preflight" | "execution";
+    reason: "exit" | "timeout" | "cancelled";
+    timeoutMs: number;
+  };
+  artifacts?: { id: string; bytes: Uint8Array }[];
+  observations?: Omit<ObservationEvidence, "source">[];
+  actualCondition?: "passive" | "enforced" | "unknown";
+  appliedInstrumentation?: InstrumentationRequest[];
+  inputTokens?: number | null;
+  outputTokens?: number | null;
+  costUsd?: number | null;
+  usageComplete?: boolean;
+}
+
+/** A failed host call can still provide bounded evidence for retention. */
+export class HostExecutionFailure extends Error {
+  readonly result: HostResult;
+  constructor(
+    message: string | undefined,
+    result: HostResult,
+    options?: ErrorOptions,
+  ) {
+    super(message, options);
+    this.result = { ...result, executionFailed: true, failureMessage: message };
+  }
+}
+
+export class TrialPersistenceError extends Error {
+  constructor(workspace: string, options?: ErrorOptions) {
+    super(
+      `trial persistence failed; fixture retained at ${workspace}`,
+      options,
+    );
+  }
+}
+
+export interface HostAdapter {
+  id: string;
+  model: string;
+  effort: string;
+  instrumentation?: InstrumentationCapability[];
+  hostCapabilities?: string[];
+  configuration?: Record<string, unknown>;
+  run(request: {
+    /** Persist private diagnostic bytes before the native host removes its state. */
+    retainHostDiagnostics?: (bytes: Uint8Array) => Promise<void>;
+    runtimePolicy?: RuntimePolicy;
+    runtimeRole?: RuntimeRole;
+    candidateTranscriptRoot?: string;
+    candidateRuntimeRoot?: string;
+    candidateArtifactRoots?: CandidateArtifactRoots;
+    prompt: string;
+    followUpPrompt?: string;
+    workspace: string;
+    condition: "passive" | "enforced";
+    fixtureBinDir?: string;
+    instrumentation?: InstrumentationRequest[];
+    codexMarketplace?: {
+      artifactRoot: string;
+      marketplaceName: string;
+      pluginNames: string[];
+      artifactPaths: string[];
+    };
+    claudePluginDirs?: {
+      artifactRoots: string[];
+      artifactPaths: string[];
+    };
+    explicitSkillInvocation?:
+      | {
+          scope?: "plugin";
+          pluginName: string;
+          skillName: string;
+          token: string;
+        }
+      | {
+          scope: "repository";
+          pluginName?: never;
+          skillName: string;
+          token: string;
+        };
+    signal?: AbortSignal;
+  }): Promise<HostResult>;
+}
+
+export interface ResolvedCase {
+  id: string;
+  prompt: string;
+  followUpPrompt?: string;
+  fixture:
+    | { files: Record<string, string>; sourceRef?: never }
+    | Omit<RepositoryFixture, "kind">
+    | (GeneratedFixture & { sourceRef?: never });
+  checks: CheckDeclaration[];
+  requiredEvidence: string[];
+}
+
+export interface EvaluationProgress {
+  stage: "preparation" | "candidate" | "grading" | "evidence";
+  status: "started" | "completed" | "skipped";
+  trial?: number;
+}
+
+export interface EvaluationOptions {
+  /** Optional best-effort observation; exceptions never change the evaluation. */
+  onProgress?: (event: EvaluationProgress) => void;
+  /** Redacted operator preview, retained for reproduction outside semantic identity. */
+  resolvedRunSettings?: Record<string, unknown>;
+  /** Additional non-secret assessment settings included in configuration identity. */
+  assessmentSettings?: Record<string, unknown>;
+  runtimePolicy?: RuntimePolicy;
+  projectRoot: string;
+  resultsRoot: string;
+  runStateRoot?: string;
+  case: ResolvedCase;
+  host: HostAdapter;
+  semanticHost?: HostAdapter;
+  advisoryHost?: HostAdapter;
+  advisoryExcludedPaths?: string[];
+  runnerBuildDigest: string;
+  runnerCheckoutRoot?: string;
+  projectDigest: string;
+  condition: "passive" | "enforced";
+  trialCount: number;
+  jobs?: number;
+  passThreshold: number;
+  dry?: boolean;
+  signal?: AbortSignal;
+  extension?: {
+    session: Awaited<ReturnType<typeof openExtensionSession>>;
+    resolvedCase: ExtensionCase;
+  };
+  preparationSources?: PreparationSources;
+  shellIsolation?: {
+    protectedRoots: string[];
+    toolchainBinDir?: string;
+    uvRuntimeCache?: boolean;
+  };
+}
+
+/** Persisted evaluations always retain a trial path and a known outcome list. */
+export interface TrialSummary
+  extends
+    Assessment,
+    Omit<
+      CliResultDataTrial,
+      keyof Assessment | "domainOutcomes" | "artifactPath"
+    > {
+  domainOutcomes: DomainOutcome[];
+  artifactPath: string;
+}
+
+export interface CaseSummary
+  extends Assessment, Omit<CliResultDataCase, keyof Assessment | "trials"> {
+  trials: TrialSummary[];
+}
+
+/** Unlike pre-run wire failures, a returned evaluation has run/evidence paths. */
+export interface CliResult
+  extends
+    Assessment,
+    Omit<
+      CliResultData,
+      keyof Assessment | "runId" | "exitCode" | "evidencePath" | "cases"
+    > {
+  runId: string;
+  exitCode: number;
+  evidencePath: string;
+  cases: CaseSummary[];
+}

package/src/evidence/host-results.ts

@@ -0,0 +1,92 @@
+import { isRecord } from "../value-guards";
+import { canonicalJson } from "../identity";
+import type { HostResult } from "../evaluation/types";
+import type {
+  ObservationCompleteness,
+  ObservationEvidence,
+} from "../contract-types";
+
+export function hostObservations(
+  result: HostResult,
+  hostId: string,
+  existingIds: Set<string>,
+): ObservationEvidence[] {
+  const items = result.observations ?? [];
+  if (items.length > 128) throw new Error("too many host observations");
+  const ids = new Set([...existingIds, "sevro.observation.final-message"]);
+  let totalBytes = 0;
+  return items.map((value: unknown) => {
+    const item = validatedHostObservation(value, ids);
+    ids.add(item.id);
+    const data = canonicalJson(item.data);
+    totalBytes += Buffer.byteLength(data, "utf8");
+    if (totalBytes > 8 * 1024 * 1024)
+      throw new Error("host observations exceed 8 MiB");
+    return {
+      id: item.id,
+      source: hostId,
+      completeness: item.completeness,
+      data: JSON.parse(data) as Record<string, unknown>,
+    };
+  });
+}
+
+export function hostArtifacts(result: HostResult, existingIds: Set<string>) {
+  const items = result.artifacts ?? [];
+  if (!Array.isArray(items) || items.length > 32)
+    throw new Error("too many host artifacts");
+  const ids = new Set(existingIds);
+  let totalBytes = 0;
+  return items.map((value: unknown) => {
+    const item = validatedHostArtifact(value, ids);
+    ids.add(item.id);
+    totalBytes += item.bytes.byteLength;
+    if (
+      item.bytes.byteLength > 8 * 1024 * 1024 ||
+      totalBytes > 32 * 1024 * 1024
+    )
+      throw new Error("host artifacts exceed the size limit");
+    return { id: item.id, bytes: Buffer.from(item.bytes) };
+  });
+}
+
+function validEvidenceId(value: unknown): value is string {
+  return (
+    typeof value === "string" &&
+    /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)+$/.test(value)
+  );
+}
+
+function validObservationId(value: unknown, ids: Set<string>): value is string {
+  return (
+    validEvidenceId(value) &&
+    !value.startsWith("sevro.observation.") &&
+    !ids.has(value)
+  );
+}
+
+function validCompleteness(value: unknown): value is ObservationCompleteness {
+  return value === "complete" || value === "partial" || value === "unavailable";
+}
+
+function validatedHostObservation(value: unknown, ids: Set<string>) {
+  if (!isRecord(value)) throw new Error("invalid host observation");
+  if (
+    !validObservationId(value.id, ids) ||
+    !validCompleteness(value.completeness) ||
+    !isRecord(value.data)
+  )
+    throw new Error("invalid host observation");
+  return { id: value.id, completeness: value.completeness, data: value.data };
+}
+
+function validArtifactId(value: unknown, ids: Set<string>): value is string {
+  return validEvidenceId(value) && !ids.has(value);
+}
+
+function validatedHostArtifact(value: unknown, ids: Set<string>) {
+  if (!isRecord(value)) throw new Error("invalid host artifact");
+  if (!validArtifactId(value.id, ids) || !(value.bytes instanceof Uint8Array))
+    throw new Error("invalid host artifact");
+  return { id: value.id, bytes: value.bytes };
+}

package/src/evidence/retained-artifacts.ts

@@ -0,0 +1,18 @@
+import { readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { sha256 } from "../identity";
+
+export async function verifyRetainedArtifacts(
+  artifacts: { path: string; sha256: string }[],
+): Promise<void> {
+  for (const artifact of artifacts) {
+    let actual: string;
+    try {
+      actual = sha256(await readFile(fileURLToPath(artifact.path)));
+    } catch {
+      throw new Error("retained preparation artifact is unreadable");
+    }
+    if (actual !== artifact.sha256)
+      throw new Error("retained preparation artifact changed");
+  }
+}

package/src/evidence/trial-writer.ts

@@ -0,0 +1,203 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import { TrialPersistenceError, type TrialSummary } from "../evaluation/types";
+import { sha256 } from "../identity";
+import type { Assessment, CheckOutcome } from "../results";
+import type {
+  ArtifactReference,
+  Diagnostic,
+  ObservationEvidence,
+  RawResultReference,
+} from "../contract-types";
+import type { RunEvidenceDataTrialEvidence } from "../schema-types";
+import { atomicWriteJson } from "../storage";
+import type { hostArtifacts } from "./host-results";
+import { verifyRetainedArtifacts } from "./retained-artifacts";
+
+type TrialEvidence = RunEvidenceDataTrialEvidence;
+export type TrialObservation = ObservationEvidence;
+
+interface TrialWriterInput {
+  readonly runDir: string;
+  readonly trial: number;
+  readonly workspace: string;
+  readonly preparationArtifacts: readonly ArtifactReference[];
+}
+
+/** Owns private trial files and their references, independently of run collection. */
+export class TrialEvidenceWriter {
+  private readonly references: ArtifactReference[];
+  private rawPath: string | null = null;
+  private rawDigest: string | null = null;
+
+  constructor(private readonly input: TrialWriterInput) {
+    this.references = [...input.preparationArtifacts];
+  }
+
+  get artifactRefs(): ArtifactReference[] {
+    return [...this.references];
+  }
+
+  get hasCandidateMessage(): boolean {
+    return this.rawDigest !== null;
+  }
+
+  private async writePrivateFile(
+    path: string,
+    bytes: string | Uint8Array,
+  ): Promise<void> {
+    try {
+      await writeFile(path, bytes, { flag: "wx", mode: 0o600 });
+    } catch (cause) {
+      throw new TrialPersistenceError(this.input.workspace, { cause });
+    }
+  }
+
+  private async retainReference(
+    id: string,
+    filename: string,
+    bytes: string | Uint8Array,
+  ): Promise<ArtifactReference> {
+    const path = join(
+      this.input.runDir,
+      `trial-${this.input.trial}-${filename}`,
+    );
+    await this.writePrivateFile(path, bytes);
+    const reference = {
+      id,
+      path: pathToFileURL(path).href,
+      sha256: sha256(bytes),
+    };
+    this.references.push(reference);
+    return reference;
+  }
+
+  async retainArtifacts(
+    artifacts: readonly ReturnType<typeof hostArtifacts>[number][],
+    prefix = "",
+  ): Promise<void> {
+    for (const artifact of artifacts)
+      await this.retainReference(
+        artifact.id,
+        `${prefix}${artifact.id}.bin`,
+        artifact.bytes,
+      );
+  }
+
+  verifyArtifacts(): Promise<void> {
+    return verifyRetainedArtifacts(this.references);
+  }
+
+  observeCandidate(
+    source: string,
+    result: { finalMessage: string | null; complete: boolean } | null,
+  ): TrialObservation {
+    if (result === null || result.finalMessage == null) {
+      this.rawDigest = null;
+      return {
+        id: "sevro.observation.final-message",
+        source,
+        completeness: "unavailable",
+        data: {},
+      };
+    }
+    this.rawDigest = sha256(result.finalMessage);
+    return {
+      id: "sevro.observation.final-message",
+      source,
+      completeness: result.complete ? "complete" : "partial",
+      data: {
+        sha256: this.rawDigest,
+        byteLength: Buffer.byteLength(result.finalMessage, "utf8"),
+      },
+    };
+  }
+
+  async retainSemanticVerdicts(
+    message: string | null,
+    suffix: string,
+  ): Promise<void> {
+    if (message === null || Buffer.byteLength(message, "utf8") > 1024 * 1024)
+      return;
+    await this.retainReference(
+      `sevro.semantic.verdicts${suffix}`,
+      `semantic-verdicts${suffix}.json`,
+      Buffer.from(message, "utf8"),
+    );
+  }
+
+  async retainAdvisoryResponse(
+    source: string,
+    message: string | null,
+  ): Promise<RawResultReference> {
+    if (message === null || Buffer.byteLength(message, "utf8") > 64 * 1024)
+      return { source, path: null, sha256: null };
+    const reference = await this.retainReference(
+      "sevro.advisory.response",
+      "advisory.json",
+      message,
+    );
+    return { source, path: reference.path, sha256: reference.sha256 };
+  }
+
+  async retainRawCandidate(message: string | null | undefined): Promise<void> {
+    await this.verifyArtifacts();
+    if (message == null) {
+      this.rawPath = null;
+      return;
+    }
+    this.rawPath = join(this.input.runDir, `trial-${this.input.trial}-raw.txt`);
+    await this.writePrivateFile(this.rawPath, message);
+  }
+
+  candidateRawResult(source: string): RawResultReference {
+    return {
+      source,
+      path: this.rawPath ? pathToFileURL(this.rawPath).href : null,
+      sha256: this.rawDigest,
+    };
+  }
+
+  diagnosticLocation(diagnostic: Diagnostic | undefined) {
+    if (!diagnostic) return diagnostic;
+    const artifact = this.references.find((ref) =>
+      ref.id.endsWith("sevro.host.diagnostics"),
+    );
+    if (!artifact) return diagnostic;
+    const message = `${diagnostic.message}; diagnostics: ${artifact.path}`;
+    return message.length <= 4096 ? { ...diagnostic, message } : diagnostic;
+  }
+
+  async persistTrial(
+    caseId: string,
+    assessment: Assessment,
+    checks: CheckOutcome[],
+    domainOutcomes: TrialSummary["domainOutcomes"],
+    evidence: TrialEvidence,
+  ): Promise<TrialSummary> {
+    const artifactPath = join(
+      this.input.runDir,
+      `trial-${this.input.trial}.json`,
+    );
+    const summary: TrialSummary = {
+      trial: this.input.trial,
+      ...assessment,
+      checks,
+      domainOutcomes,
+      artifactPath,
+    };
+    try {
+      await atomicWriteJson(artifactPath, {
+        format: "sevro.trial-evidence.v1",
+        caseId,
+        trial: this.input.trial,
+        result: summary,
+        evidence,
+      });
+    } catch (cause) {
+      throw new TrialPersistenceError(this.input.workspace, { cause });
+    }
+    return summary;
+  }
+}

package/src/evidence/usage.ts

@@ -0,0 +1,17 @@
+import type { HostResult } from "../evaluation/types";
+import type { UsageEvidence } from "../contract-types";
+
+export function usage(result: HostResult | null): UsageEvidence {
+  return result === null
+    ? { inputTokens: null, outputTokens: null, costUsd: null, complete: false }
+    : availableUsage(result);
+}
+
+function availableUsage(result: HostResult): UsageEvidence {
+  return {
+    inputTokens: result.inputTokens ?? null,
+    outputTokens: result.outputTokens ?? null,
+    costUsd: result.costUsd ?? null,
+    complete: result.usageComplete ?? false,
+  };
+}