@bjoernrochel/sevro 0.1.0-rc.8 → 1.0.0

package/src/evaluation-fixture.ts

@@ -1,302 +0,0 @@
-import { isRecord } from "./value-guards";
-import { sha256 } from "./identity";
-import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
-import { allocateWorkspace, removeWorkspace } from "./workspace-state";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-import { canonicalJson } from "./identity";
-import { fixtureBinDirectory, installFixtureBin } from "./fixture-bin";
-import { installGitHooks } from "./git-hooks";
-import {
-  materializeGeneratedFixture,
-  type GeneratedFixture,
-} from "./generated-fixture";
-import { runFixtureSetup, type FixtureSetup } from "./fixture-setup";
-import {
-  fixtureParts,
-  safePreparationTarget,
-  type InlineArtifact,
-  type PreparationSources,
-} from "./preparation";
-import {
-  applyRepositoryOverlay,
-  cloneRepositorySource,
-  type RepositoryFixture,
-  type RepositorySource,
-} from "./repository-fixture";
-import {
-  EvaluationConfigurationError,
-  type HostResult,
-  type ResolvedCase,
-} from "./evaluation-types";
-
-export { sha256 } from "./identity";
-
-export async function createFixture(
-  fixture: ResolvedCase["fixture"],
-  artifacts: InlineArtifact[],
-  sources: PreparationSources | undefined,
-  repository: RepositorySource | null,
-  repositoryFixture: RepositoryFixture | null,
-  generated: GeneratedFixture | null,
-  setup: FixtureSetup | null,
-  projectRoot: string,
-  signal?: AbortSignal,
-  reservedWorkspace?: string,
-): Promise<string> {
-  const paths = inlineFixturePaths(fixture, generated, repository);
-  const workspace = reservedWorkspace ?? (await allocateWorkspace());
-  try {
-    await initializeFixtureHistory(
-      fixture,
-      sources,
-      repository,
-      repositoryFixture,
-      generated,
-      workspace,
-    );
-    await installDeclaredFixtureTools(
-      generated ?? repositoryFixture,
-      workspace,
-    );
-    await writeInlineFixtureFiles(paths, workspace);
-    await setupFixture(setup, workspace, projectRoot, signal);
-    await mountPreparationArtifacts(artifacts, workspace);
-    await excludePreparationArtifacts(artifacts, workspace);
-    await fixtureBinDirectory(workspace);
-    return workspace;
-  } catch (error) {
-    if (!reservedWorkspace) await removeWorkspace(workspace);
-    throw error;
-  }
-}
-
-function inlineFixturePaths(
-  fixture: ResolvedCase["fixture"],
-  generated: GeneratedFixture | null,
-  repository: RepositorySource | null,
-) {
-  return Object.entries(
-    generated || repository ? {} : (fixture.files ?? {}),
-  ).map(([path, content]) => ({ path, parts: fixtureParts(path), content }));
-}
-
-async function cloneFixtureRepository(
-  fixture: ResolvedCase["fixture"],
-  sources: PreparationSources | undefined,
-  repository: RepositorySource,
-  workspace: string,
-): Promise<void> {
-  if (!fixture.sourceRef || !sources)
-    throw new EvaluationConfigurationError(
-      "repository source root is required and must be absolute",
-    );
-  await cloneRepositorySource(
-    fixture.sourceRef,
-    sources,
-    repository,
-    workspace,
-  );
-}
-
-async function initializeFixtureHistory(
-  fixture: ResolvedCase["fixture"],
-  sources: PreparationSources | undefined,
-  repository: RepositorySource | null,
-  repositoryFixture: RepositoryFixture | null,
-  generated: GeneratedFixture | null,
-  workspace: string,
-): Promise<void> {
-  if (repository)
-    await cloneFixtureRepository(fixture, sources, repository, workspace);
-  if (generated) await materializeGeneratedFixture(generated, workspace);
-  if (repositoryFixture)
-    await applyRepositoryOverlay(repositoryFixture, workspace);
-}
-
-async function installDeclaredFixtureTools(
-  fixture: GeneratedFixture | RepositoryFixture | null,
-  workspace: string,
-): Promise<void> {
-  await installFixtureBin(fixture?.bin, workspace);
-  await installGitHooks(fixture?.hooks, workspace);
-}
-
-async function writeInlineFixtureFiles(
-  files: ReturnType<typeof inlineFixturePaths>,
-  workspace: string,
-): Promise<void> {
-  for (const file of files) {
-    const target = join(workspace, ...file.parts);
-    await mkdir(dirname(target), { recursive: true, mode: 0o700 });
-    await writeFile(target, file.content, { flag: "wx", mode: 0o600 });
-  }
-}
-
-async function setupFixture(
-  setup: FixtureSetup | null,
-  workspace: string,
-  projectRoot: string,
-  signal?: AbortSignal,
-): Promise<void> {
-  if (!setup) return;
-  await runFixtureSetup(setup, {
-    workspace,
-    projectRoot,
-    fixtureBinDir: (await fixtureBinDirectory(workspace)) ?? undefined,
-    signal,
-  });
-}
-
-async function mountPreparationArtifacts(
-  artifacts: InlineArtifact[],
-  workspace: string,
-): Promise<void> {
-  for (const artifact of artifacts) {
-    const target = await safePreparationTarget(
-      workspace,
-      artifact.relativePath,
-    );
-    await writeFile(target, artifact.bytes, {
-      flag: "wx",
-      mode: artifact.executable ? 0o700 : 0o600,
-    });
-  }
-}
-
-async function excludePreparationArtifacts(
-  artifacts: InlineArtifact[],
-  workspace: string,
-): Promise<void> {
-  const gitExcluded = artifacts.filter((artifact) => artifact.gitExclude);
-  if (!gitExcluded.length) return;
-  const special = new Set(["\\", "*", "?", "[", "]", "#", "!", " "]);
-  const patterns = gitExcluded.map((artifact) =>
-    Array.from(artifact.relativePath)
-      .map((character) =>
-        special.has(character) ? `\\${character}` : character,
-      )
-      .join(""),
-  );
-  await appendFile(
-    join(workspace, ".git", "info", "exclude"),
-    `\n${patterns.map((path) => `/${path}`).join("\n")}\n`,
-  );
-}
-
-export function hostObservations(
-  result: HostResult,
-  hostId: string,
-  existingIds: Set<string>,
-) {
-  const items = result.observations ?? [];
-  if (items.length > 128) throw new Error("too many host observations");
-  const ids = new Set([...existingIds, "sevro.observation.final-message"]);
-  let totalBytes = 0;
-  return items.map((value: unknown) => {
-    const item = validatedHostObservation(value, ids);
-    ids.add(item.id);
-    const data = canonicalJson(item.data);
-    totalBytes += Buffer.byteLength(data, "utf8");
-    if (totalBytes > 8 * 1024 * 1024)
-      throw new Error("host observations exceed 8 MiB");
-    return {
-      id: item.id,
-      source: hostId,
-      completeness: item.completeness,
-      data: JSON.parse(data) as Record<string, unknown>,
-    };
-  });
-}
-
-export function hostArtifacts(result: HostResult, existingIds: Set<string>) {
-  const items = result.artifacts ?? [];
-  if (!Array.isArray(items) || items.length > 32)
-    throw new Error("too many host artifacts");
-  const ids = new Set(existingIds);
-  let totalBytes = 0;
-  return items.map((value: unknown) => {
-    const item = validatedHostArtifact(value, ids);
-    ids.add(item.id);
-    totalBytes += item.bytes.byteLength;
-    if (
-      item.bytes.byteLength > 8 * 1024 * 1024 ||
-      totalBytes > 32 * 1024 * 1024
-    )
-      throw new Error("host artifacts exceed the size limit");
-    return { id: item.id, bytes: Buffer.from(item.bytes) };
-  });
-}
-
-function validEvidenceId(value: unknown): value is string {
-  return (
-    typeof value === "string" &&
-    /^[a-z][a-z0-9]*(?:[.-][a-z0-9]+)+$/.test(value)
-  );
-}
-
-function validObservationId(value: unknown, ids: Set<string>): value is string {
-  return (
-    validEvidenceId(value) &&
-    !value.startsWith("sevro.observation.") &&
-    !ids.has(value)
-  );
-}
-
-function validCompleteness(
-  value: unknown,
-): value is "complete" | "partial" | "unavailable" {
-  return value === "complete" || value === "partial" || value === "unavailable";
-}
-
-function validatedHostObservation(value: unknown, ids: Set<string>) {
-  if (!isRecord(value)) throw new Error("invalid host observation");
-  if (
-    !validObservationId(value.id, ids) ||
-    !validCompleteness(value.completeness) ||
-    !isRecord(value.data)
-  )
-    throw new Error("invalid host observation");
-  return { id: value.id, completeness: value.completeness, data: value.data };
-}
-
-function validArtifactId(value: unknown, ids: Set<string>): value is string {
-  return validEvidenceId(value) && !ids.has(value);
-}
-
-function validatedHostArtifact(value: unknown, ids: Set<string>) {
-  if (!isRecord(value)) throw new Error("invalid host artifact");
-  if (!validArtifactId(value.id, ids) || !(value.bytes instanceof Uint8Array))
-    throw new Error("invalid host artifact");
-  return { id: value.id, bytes: value.bytes };
-}
-
-export async function verifyRetainedArtifacts(
-  artifacts: { path: string; sha256: string }[],
-): Promise<void> {
-  for (const artifact of artifacts) {
-    let actual: string;
-    try {
-      actual = sha256(await readFile(fileURLToPath(artifact.path)));
-    } catch {
-      throw new Error("retained preparation artifact is unreadable");
-    }
-    if (actual !== artifact.sha256)
-      throw new Error("retained preparation artifact changed");
-  }
-}
-
-export function usage(result: HostResult | null) {
-  return result === null
-    ? { inputTokens: null, outputTokens: null, costUsd: null, complete: false }
-    : availableUsage(result);
-}
-
-function availableUsage(result: HostResult) {
-  return {
-    inputTokens: result.inputTokens ?? null,
-    outputTokens: result.outputTokens ?? null,
-    costUsd: result.costUsd ?? null,
-    complete: result.usageComplete ?? false,
-  };
-}

package/src/evaluation-run.ts

@@ -1,200 +0,0 @@
-import { prepareEvaluation } from "./evaluation-plan";
-import {
-  type CaseSummary,
-  type CliResult,
-  type EvaluationOptions,
-} from "./evaluation-types";
-import { canonicalJson, hashJson } from "./identity";
-import {
-  exitCodeFor,
-  summarizeAssessments,
-  summarizeCases,
-  type Assessment,
-} from "./results";
-import { runtimePolicySnapshot } from "./runtime-config";
-import { cleanupExpiredWorkspaces } from "./workspace-state";
-import { assertCliResult, assertRunEvidence } from "./schema";
-import { atomicWriteJson } from "./storage";
-import {
-  allocateEvaluationRun,
-  checkpointEvaluation,
-  cleanupEvaluationWorkspaces,
-  diagnoseEvaluationFailure,
-  reserveEvaluationWorkspaces,
-  retainPreparationArtifacts,
-  type EvaluationContext,
-} from "./evaluation-context";
-import { executeEvaluationTrials } from "./evaluation-trial";
-
-function evaluationCaseSummary(
-  options: EvaluationOptions,
-  context: EvaluationContext,
-): CaseSummary {
-  const assessment: Assessment =
-    !context.trialSummaries.length && options.signal?.aborted
-      ? {
-          execution: { status: "cancelled" },
-          grading: { status: "not_requested" },
-          task: { verdict: "not_assessed" },
-        }
-      : summarizeAssessments(context.trialSummaries, options.passThreshold);
-  return {
-    caseId: options.case.id,
-    ...assessment,
-    trials: context.trialSummaries,
-  };
-}
-
-function evaluationSignal(
-  options: EvaluationOptions,
-): "SIGINT" | "SIGTERM" | undefined {
-  const reason: unknown = options.signal?.aborted
-    ? options.signal.reason
-    : null;
-  return reason === "SIGINT" || reason === "SIGTERM" ? reason : undefined;
-}
-
-function finalCliResult(
-  options: EvaluationOptions,
-  context: EvaluationContext,
-): CliResult {
-  const caseResult = evaluationCaseSummary(options, context);
-  const assessment = summarizeCases([caseResult]);
-  const diagnostic = context.diagnostics.sort(
-    (left, right) => left.trial - right.trial,
-  )[0]?.diagnostic;
-  return {
-    format: "sevro.cli-result.v1",
-    runId: context.runId,
-    ...assessment,
-    exitCode: exitCodeFor(assessment, {
-      dryRun: options.dry,
-      signal: evaluationSignal(options),
-    }),
-    evidencePath: context.evidencePath,
-    cases: [caseResult],
-    ...(diagnostic ? { diagnostic } : {}),
-  };
-}
-
-function finalConditionEvidence(
-  options: EvaluationOptions,
-  context: EvaluationContext,
-) {
-  const actualConditions = [
-    ...new Set(context.trialEvidence.map((entry) => entry.condition.actual)),
-  ];
-  const commonAppliedInstrumentation =
-    context.trialEvidence.length === options.trialCount &&
-    context.trialEvidence.every(
-      (entry) =>
-        canonicalJson(entry.condition.appliedInstrumentation) ===
-        canonicalJson(context.requestedInstrumentation),
-    )
-      ? context.requestedInstrumentation
-      : [];
-  return {
-    requested: options.condition,
-    actual: actualConditions.length === 1 ? actualConditions[0] : "unknown",
-    requestedInstrumentation: context.requestedInstrumentation,
-    appliedInstrumentation: commonAppliedInstrumentation,
-  };
-}
-
-function finalExtensionEvidence(
-  options: EvaluationOptions,
-  replacedBuiltinGraders: string[],
-) {
-  const extension = options.extension;
-  if (!extension) return null;
-  const identity = extension.session.identity;
-  return {
-    id: identity.id,
-    version: identity.version,
-    sourceDigest: identity.sourceDigest,
-    configurationDigest: identity.configurationDigest,
-    protocol: identity.protocol,
-    capabilities: identity.capabilities,
-    replacements: {
-      graders: replacedBuiltinGraders,
-      taskVerdictPolicy: identity.selectedTaskVerdictPolicy,
-    },
-  };
-}
-
-function finalRunEvidence(
-  options: EvaluationOptions,
-  context: EvaluationContext,
-  result: CliResult,
-  diagnostic: { code: string; message: string } | undefined,
-) {
-  const condition = finalConditionEvidence(options, context);
-  return {
-    format: "sevro.run-evidence.v1",
-    runId: context.runId,
-    evaluationIdentity: context.evaluationIdentity,
-    configuration: {
-      digest: hashJson(context.redactedConfig),
-      redacted: context.redactedConfig,
-    },
-    runner: context.runner,
-    project: context.project,
-    extension: finalExtensionEvidence(options, context.replacedBuiltinGraders),
-    condition,
-    graders: {
-      active: context.activeGraders,
-      replacedDefaults: context.replacedBuiltinGraders,
-    },
-    routes: context.routes,
-    result,
-    trials: context.trialEvidence,
-    ...(diagnostic ? { diagnostic } : {}),
-  };
-}
-
-async function finishEvaluationRun(
-  options: EvaluationOptions,
-  context: EvaluationContext,
-) {
-  const diagnostic = context.diagnostics.sort(
-    (left, right) => left.trial - right.trial,
-  )[0]?.diagnostic;
-  const result = finalCliResult(options, context);
-  const evidence = finalRunEvidence(options, context, result, diagnostic);
-  assertCliResult(result);
-  assertRunEvidence(evidence);
-  await atomicWriteJson(context.evidencePath, evidence);
-  checkpointEvaluation(
-    context,
-    options.signal?.aborted ? "interrupted" : "complete",
-  );
-  return { result };
-}
-
-/** Execute resolved evaluator data through an injected host and retain every trial. */
-export async function runEvaluation(
-  options: EvaluationOptions,
-): Promise<{ result: CliResult }> {
-  options = {
-    ...options,
-    runtimePolicy: runtimePolicySnapshot(options.runtimePolicy),
-  };
-  await cleanupExpiredWorkspaces();
-  const plan = await prepareEvaluation(options);
-  const runtime = await allocateEvaluationRun(options, plan);
-  try {
-    const artifacts = await retainPreparationArtifacts({ ...plan, ...runtime });
-    const records = await reserveEvaluationWorkspaces(options, {
-      ...plan,
-      ...runtime,
-    });
-    const context = { ...plan, ...runtime, ...artifacts, ...records };
-    await executeEvaluationTrials(options, context);
-    return await finishEvaluationRun(options, context);
-  } catch (error) {
-    diagnoseEvaluationFailure(runtime);
-    throw error;
-  } finally {
-    await cleanupEvaluationWorkspaces(runtime);
-  }
-}